### Add partial first draft for jsh standards document Ian Adam Naval 5 years ago
parent
commit
7d34bcae51
2 changed files with 237 additions and 0 deletions
1. 7
0
.gitignore
2. 230
0
jsh_standard.tex

#### + 7 - 0 .gitignoreView File

 @@ -0,0 +1,7 @@ 1 +*.aux 2 +*.dvi 3 +*.log 4 +*.out 5 +*.pdf 6 +*.gz 7 +*.log

#### + 230 - 0 jsh_standard.texView File

 @@ -0,0 +1,230 @@ 1 +%---------------------------------------------------------------------------------------- 2 +% DOCUMENT CONFIGURATION 3 +%---------------------------------------------------------------------------------------- 4 + 5 +\documentclass[twoside]{article} 6 + 7 +\usepackage{lipsum} % Package to generate dummy text throughout this template 8 + 9 +\usepackage[sc]{mathpazo} % Use the Palatino font 10 +\usepackage[T1]{fontenc} % Use 8-bit encoding that has 256 glyphs 11 +\linespread{1.05} % Line spacing - Palatino needs more space between lines 12 +\usepackage{microtype} % Slightly tweak font spacing for aesthetics 13 + 14 +\usepackage[hmarginratio=1:1,top=32mm,columnsep=20pt]{geometry} % Document margins 15 +\usepackage{multicol} % Used for the two-column layout of the document 16 +\usepackage[hang, small,labelfont=bf,up,textfont=it,up]{caption} % Custom captions under/above floats in tables or figures 17 +\usepackage{booktabs} % Horizontal rules in tables 18 +\usepackage{float} % Required for tables and figures in the multi-column environment - they need to be placed in specific locations with the [H] (e.g. \begin{table}[H]) 19 +\usepackage{hyperref} % For hyperlinks in the PDF 20 + 21 +\usepackage{lettrine} % The lettrine is the first enlarged letter at the beginning of the text 22 +\usepackage{paralist} % Used for the compactitem environment which makes bullet points with less space between them 23 + 24 +\usepackage{abstract} % Allows abstract customization 25 +\renewcommand{\abstractnamefont}{\normalfont\bfseries} % Set the "Abstract" text to bold 26 +\renewcommand{\abstracttextfont}{\normalfont\small\itshape} % Set the abstract itself to small italic text 27 + 28 +\usepackage{titlesec} % Allows customization of titles 29 +\renewcommand\thesection{\Roman{section}} % Roman numerals for the sections 30 +\renewcommand\thesubsection{\Roman{subsection}} % Roman numerals for subsections 31 +\titleformat{\section}[block]{\large\scshape\centering}{\thesection.}{1em}{} % Change the look of the section titles 32 +\titleformat{\subsection}[block]{\large}{\thesubsection.}{1em}{} % Change the look of the section titles 33 + 34 +\usepackage{fancyhdr} % Headers and footers 35 +\pagestyle{fancy} % All pages have headers and footers 36 +\fancyhead{} % Blank out the default header 37 +\fancyfoot{} % Blank out the default footer 38 +%\fancyhead[C]{Running title $\bullet$ November 2012 $\bullet$ Vol. XXI, No. 1} % Custom header text 39 +\fancyfoot[RO,LE]{\thepage} % Custom footer text 40 + 41 +\newcommand{\code}{\texttt{#1}} 42 + 43 +%---------------------------------------------------------------------------------------- 44 +% TITLE SECTION 45 +%---------------------------------------------------------------------------------------- 46 + 47 +\title{\vspace{-15mm}\fontsize{24pt}{10pt}\selectfont\textbf{Standards for Input and Output in the Augmented Unix Userland}} % Article title 48 + 49 +\author{ 50 +\large 51 +\textsc{Sam Abradi, Ian Naval, Fredric Silberberg}\\[2mm] % Your name 52 +\normalsize Worcester Polytechnic Institute \\ % Your institution 53 +\normalsize \{\href{mailto:stabradi@wpi.edu}{stabradi},\href{mailto:ianaval@wpi.edu}{ianaval},\href{mailto:fbsilberberg@wpi.edu}{fbsilberberg}\}@wpi.edu % Your email address 54 +\vspace{-5mm} 55 +} 56 +\date{} 57 + 58 +%---------------------------------------------------------------------------------------- 59 + 60 +\begin{document} 61 + 62 +\maketitle % Insert title 63 + 64 +\thispagestyle{fancy} % All pages have headers and footers 65 + 66 +%---------------------------------------------------------------------------------------- 67 +% ABSTRACT 68 +%---------------------------------------------------------------------------------------- 69 + 70 +\begin{abstract} 71 + 72 +\noindent The Augmented Unix Userland includes a set of standards to which all modified utility programs must adhere. This document outlines these guidelines and provides detailed explanations for each standard. 73 + 74 +\end{abstract} 75 + 76 +%---------------------------------------------------------------------------------------- 77 +% ARTICLE CONTENTS 78 +%---------------------------------------------------------------------------------------- 79 + 80 +\begin{multicols}{2} % Two-column layout throughout the main article text 81 + 82 +\section{Background} 83 + 84 +[Insert general introduction and talking about how we need to define some terminology such as streams, substreams and the tributary utility] 85 + 86 +\subsection{Streams} 87 + 88 +We are taking the traditional concept of streams and sending updates from program to program in frames. This allows us to easily enable parallel processing by turning standard input and output into constant streams of data instead of having to buffer all of the output from one program, wait for it to finish, and then pass it along to the next program. We do this by having the output of a program be an array of output blocks, each with its own StdOut and StdErr. These are arrays themselves, and each named object becomes a substream. In the example below, the StdOut stream would have a substream called processes, and each element of the stream would be passed as the frame comes.  89 + 90 +\subsection{Tributary Utilty} 91 + 92 +This format allows us to define a new utility tentatively called tributary'' that allows you to query substreams from the toplevel standard output format. For example, running it with the query "stdout" is basically equivalent to echo''. But if we query for the processes'' subfield using the syntax stdout>processes'', then we can pull out all the values corresponding to the objects containing a processes'' key in StdOut. Think of it as grep for JSON but built around the concept of using keys and values to navigate output. 93 + 94 +\section{Standards} 95 + 96 +[Insert details about the standardized format] 97 + 98 +Things to note: 99 +\begin{compactitem} 100 +\item All output must be standard JSON as defined by RFC 7159. 101 +\item The top-level JSON value must be a standard JSON array as defined in section 5 of RFC 7159. 102 +\item The elements of the top level array (henceforth called JSH frames) must be standard JSON objects as defined in section 4 of RFC 7159. 103 +\item JSH frames must contain two members whose names are StdOut'' and StdErr'' and whose values are standard JSON arrays. 104 +\item A top-level JSH stream is an array of standard JSON values. The concatenation of arrays from the JSH frames constitute the elements of a top-level JSH stream. Only the arrays whose associated names are the same across all frames. 105 +\item A substream is an array of standard JSON values. A substream has a parent stream and an associated name. The substreams elements are derived by iterating over the elements of the parent stream. For each element of the parent stream, the following process is applied. If the element is an array, then its values are concatenated to the substream. If the element is an object and contains a value corresponding to the substream's associated name, then that value is added to the substream. If the element is neither an object nor an array, then the element itself is added to the substream. 106 +\end{compactitem} 107 + 108 +\subsection{Tributary Selectors} 109 + 110 +We also decided that any valid JSON key name is a valid key for a substream selector. However, if the name contains a space character, a ">", or a single quotation mark symbol, we require that the entire key be surrounded with quotation marks. Using single quotation marks within the selector requires that they be escaped with a backslash. 111 + 112 +\section{Examples and Use Cases} 113 + 114 +\subsection{Example: ps} 115 +For example, the program \code{ps}'s output could look like either Figure~\ref{fig:psSingleFrame} or Figure~\ref{fig:psMultipleFrame}. Both forms of output are equivalent. 116 + 117 +\begin{figure}[H] 118 +\begin{verbatim} 119 +[{ 120 + "StdOut": [ 121 + {"processes":  122 + {"pid": 1, "name": "init"}}, 123 + {"processes":  124 + {"pid": 2, "name": "bash"}}, 125 + {"processes":  126 + {"pid": 3, "name": "ps"}} 127 + ], 128 + "StdErr": [] 129 +}]\end{verbatim} 130 +\caption{Sample Output of ps with a Single Frame} 131 +\label{fig:psSingleFrame} 132 +\end{figure} 133 +\begin{figure}[H] 134 +\begin{verbatim} 135 +[{ 136 + "StdOut": [ 137 + {"processes":  138 + {"pid": 1, "name": "init"}}, 139 + {"processes":  140 + {"pid": 2, "name": "bash"}} 141 + ], 142 + "StdErr": [] 143 +}, 144 +{ 145 + "StdOut": [ 146 + {"processes":  147 + {"pid": 3, "name": "ps"}} 148 + ], 149 + "StdErr": [] 150 +}]\end{verbatim} 151 +\caption{Sample Output of ps with Multiple Frames} 152 +\label{fig:psMultipleFrame} 153 +\end{figure} 154 + 155 + 156 +\begin{figure}[H] 157 +\begin{verbatim} 158 +$ps | tributary "stdout>processes" 159 +[{ 160 + "StdOut": [ 161 + {"pid": 1, "name": "init"}, 162 + {"pid": 2, "name": "bash"}, 163 + {"pid": 3, "name": "ps"} 164 + ], 165 + "StdErr": [] 166 +}]\end{verbatim} 167 +\caption{Sample Output of ps Piped to Tributary} 168 +\label{fig:psWithTributary} 169 +\end{figure} 170 +In future weeks, we will probably hide the details of using the tributary program with special syntax in our new shell language. 171 + 172 + 173 +\begin{figure}[H] 174 +\begin{verbatim} 175 +$ nonstandard_ls 176 +[{ 177 + "StdOut": [ 178 + { 179 + "foo": { 180 + "files": [{"name": ____}] 181 + } 182 + } 183 + ], 184 + "StdErr": [] 185 +}] 186 +\end{verbatim} 187 +\caption{Sample Output of Nonstandard ls} 188 +\label{fig:nonstandardLs} 189 +\end{figure} 190 + 191 +\begin{figure}[H] 192 +\begin{verbatim} 193 +[{ 194 + "StdOut": [ 195 + { 196 + "files": [{"name": ____}, ...] 197 + } 198 + ], 199 + "Stderr": [] 200 +}] 201 +\end{verbatim} 202 +\caption{Expected Input Template for Standard Cat} 203 +\label{fig:expectedInputForStandardCat} 204 +\end{figure} 205 + 206 +Piping out the nonstandard format of ls through the \code{tributary} program would allow the user to normalize the data by simply typing \code{nonstandard\_ls | tributary "stdout>foo" | cat}.'' 207 + 208 +\section{Using Tributary to Standardize Output} 209 + 210 +While there is a tentatively defined format for all of our first party utilities, the expectation is that others will also use our output format when writing new utilities and may not always follow the spec to the letter. If some third party application nests some kernel of standardized output within fields that are unknown to any of the standard utilities, using our tributary function will enable users to easily access that standard info, and essentially peel it out into its own stream. 211 +\cite{jsonRFC} 212 + 213 +% REFERENCE LIST 214 +%---------------------------------------------------------------------------------------- 215 + 216 +\begin{thebibliography}{99} % Bibliography - this is intentionally simple in this template 217 + 218 +\bibitem[RFC 7159]{jsonRFC} 219 +T. Bray, Ed. 220 +\newblock The JavaScript Object Notation (JSON) Data Interchange Format. 221 +\newblock Internet Engineering Task Force (IETF). 222 +  223 +\end{thebibliography} 224 + 225 +%---------------------------------------------------------------------------------------- 226 + 227 + 228 +\end{multicols} 229 + 230 +\end{document}
{"ft_lang_label":"__label__en","ft_lang_prob":0.5409331,"math_prob":0.8906847,"size":12686,"snap":"2019-51-2020-05","text_gpt3_token_len":3688,"char_repetition_ratio":0.22693582,"word_repetition_ratio":0.04724062,"special_character_ratio":0.45254612,"punctuation_ratio":0.095183484,"nsfw_num_words":0,"has_unicode_error":false,"math_prob_llama3":0.95490247,"pos_list":[0,1,2],"im_url_duplicate_count":[null,null,null],"WARC_HEADER":"{\"WARC-Type\":\"response\",\"WARC-Date\":\"2020-01-23T05:31:08Z\",\"WARC-Record-ID\":\"<urn:uuid:906829de-e3c7-4bea-9b27-ecf0b58bde51>\",\"Content-Length\":\"112038\",\"Content-Type\":\"application/http; msgtype=response\",\"WARC-Warcinfo-ID\":\"<urn:uuid:64269cd8-93a0-4de0-9b4c-8c89dceea3cc>\",\"WARC-Concurrent-To\":\"<urn:uuid:c1158ee5-69d9-4f15-968c-1a757a1ff558>\",\"WARC-IP-Address\":\"157.230.67.9\",\"WARC-Target-URI\":\"https://git.ianonavy.com/MQP/notes/commit/7d34bcae511a17b1fac70fee2a3d006add45a72c\",\"WARC-Payload-Digest\":\"sha1:G7X2QWCNNK2TT4H7GCK5RPEQRQTVEOBT\",\"WARC-Block-Digest\":\"sha1:EOB7XSAFGZ3AUF7NIE7VXWSHPO52GEM5\",\"WARC-Identified-Payload-Type\":\"text/html\",\"warc_filename\":\"/cc_download/warc_2020/CC-MAIN-2020-05/CC-MAIN-2020-05_segments_1579250608295.52_warc_CC-MAIN-20200123041345-20200123070345-00466.warc.gz\"}"}