To compile documentation from the textual syntax in an extension into a tree.
§1. We will actually wrap the result in the following structure, but it's really not much more than the tree produced by Documentation Tree:
typedef struct compiled_documentation { struct text_stream *title; struct text_stream *original; struct inform_extension *associated_extension; struct inform_extension *within_extension; struct heterogeneous_tree *tree; int total_headings[3]; int total_examples; int empty; struct linked_list *cases; of satellite_test_case CLASS_DEFINITION } compiled_documentation; compiled_documentation *DocumentationCompiler::new_wrapper(text_stream *source) { compiled_documentation *cd = CREATE(compiled_documentation); cd->title = Str::new(); cd->original = Str::duplicate(source); cd->associated_extension = NULL; cd->within_extension = NULL; cd->tree = DocumentationTree::new(); cd->total_examples = 0; cd->total_headings[0] = 1; cd->total_headings[1] = 0; cd->total_headings[2] = 0; cd->empty = FALSE; cd->cases = NEW_LINKED_LIST(satellite_test_case); return cd; } typedef struct satellite_test_case { int is_example; struct text_stream *owning_heading; struct tree_node *owning_node; struct compiled_documentation *owner; struct text_stream *short_name; struct filename *test_file; struct filename *ideal_transcript; CLASS_DEFINITION } satellite_test_case;
- The structure compiled_documentation is accessed in 2/wrk, 2/edt, 2/cps, 2/rqr, 2/jm, 3/bg, 3/is, 4/ebm, 4/km, 4/lm, 4/pm, 4/pbm, 4/pfm, 4/tm, 5/es, 5/ks, 5/ls, 5/ps2, 6/inc, 7/tm, 7/eip, 7/ti, 7/tc, 7/dr and here.
- The structure satellite_test_case is accessed in 5/ks and here.
§2. We can compile either from a file...
compiled_documentation *DocumentationCompiler::compile_from_path(pathname *P, inform_extension *associated_extension) { filename *F = Filenames::in(P, I"Documentation.txt"); if (TextFiles::exists(F) == FALSE) return NULL; compiled_documentation *cd = DocumentationCompiler::compile_from_file(F, associated_extension); if (cd == NULL) return NULL; pathname *EP = Pathnames::down(P, I"Examples"); int egs = TRUE; Scan EP directory for examples2.1; egs = FALSE; EP = Pathnames::down(P, I"Tests"); Scan EP directory for examples2.1; return cd; }
§2.1. Scan EP directory for examples2.1 =
scan_directory *D = Directories::open(EP); if (D) { TEMPORARY_TEXT(leafname) while (Directories::next(D, leafname)) { wchar_t first = Str::get_first_char(leafname), last = Str::get_last_char(leafname); if (Platform::is_folder_separator(last)) continue; if (first == '.') continue; if (first == '(') continue; text_stream *short_name = Str::new(); filename *F = Filenames::in(EP, leafname); Filenames::write_unextended_leafname(short_name, F); if ((Str::get_at(short_name, Str::len(short_name)-2) == '-') && ((Str::get_at(short_name, Str::len(short_name)-1) == 'I') || (Str::get_at(short_name, Str::len(short_name)-1) == 'i'))) continue; satellite_test_case *stc = CREATE(satellite_test_case); stc->is_example = egs; stc->owning_heading = NULL; stc->owning_node = NULL; stc->owner = cd; stc->short_name = short_name; stc->test_file = F; stc->ideal_transcript = NULL; TEMPORARY_TEXT(ideal_leafname) WRITE_TO(ideal_leafname, "%S-I.txt", stc->short_name); filename *IF = Filenames::in(EP, ideal_leafname); if (TextFiles::exists(IF)) stc->ideal_transcript = IF; DISCARD_TEXT(ideal_leafname) if (stc->is_example) { Scan the example for its header and content2.1.2; } ADD_TO_LINKED_LIST(stc, satellite_test_case, cd->cases); } DISCARD_TEXT(leafname) Directories::close(D); }
- This code is used in §2 (twice).
typedef struct example_scanning_state { int star_count; struct text_stream *long_title; struct text_stream *body_text; struct text_stream *placement; struct text_stream *desc; struct linked_list *errors; struct heterogeneous_tree *tree; struct text_stream *scanning; int past_header; } example_scanning_state;
- The structure example_scanning_state is accessed in 5/es, 7/tc, 7/dt, 7/dr and here.
§2.1.2. Scan the example for its header and content2.1.2 =
example_scanning_state ess; ess.star_count = 1; ess.long_title = NULL; ess.body_text = Str::new(); ess.placement = NULL; ess.desc = NULL; ess.errors = NEW_LINKED_LIST(tree_node); ess.tree = cd->tree; ess.past_header = FALSE; ess.scanning = Str::new(); WRITE_TO(ess.scanning, "%S", Filenames::get_leafname(stc->test_file)); TextFiles::read(stc->test_file, FALSE, "unable to read file of example", TRUE, &DocumentationCompiler::read_example_helper, NULL, &ess); tree_node *placement_node = NULL; if (Str::len(ess.placement) == 0) { placement_node = cd->tree->root; } else { placement_node = DocumentationTree::find_section(cd->tree, ess.placement); if (placement_node == NULL) { DocumentationCompiler::example_error(&ess, I"example gives a Location which is not the name of any section"); } } if (Str::len(ess.desc) == 0) { DocumentationCompiler::example_error(&ess, I"example does not give its Description"); } tree_node *content_node = NULL; if (Str::len(ess.body_text) == 0) { DocumentationCompiler::example_error(&ess, I"example does not give any actual content"); } else { compiled_documentation *ecd = DocumentationCompiler::compile(ess.body_text, associated_extension); if ((ecd) && (ecd->tree) && (ecd->tree->root) && (ecd->tree->root->child) && (ecd->tree->root->child->type == passage_TNT)) content_node = ecd->tree->root->child; else { DocumentationCompiler::example_error(&ess, I"example file content is missing or wrongly set out"); } } tree_node *example_node = DocumentationTree::new_example(cd->tree, ess.long_title, ess.desc, ess.star_count, ++(cd->total_examples)); if (placement_node) Trees::make_child(example_node, placement_node); if (content_node) Trees::make_child(content_node, example_node); tree_node *E; LOOP_OVER_LINKED_LIST(E, tree_node, ess.errors) Trees::make_child(E, cd->tree->root);
- This code is used in §2.1.
void DocumentationCompiler::example_error(example_scanning_state *ess, text_stream *text) { text_stream *err = Str::new(); WRITE_TO(err, "Example file '%S': %S", ess->scanning, text); tree_node *E = DocumentationTree::new_source_error(ess->tree, err); ADD_TO_LINKED_LIST(E, tree_node, ess->errors); }
void DocumentationCompiler::read_example_helper(text_stream *text, text_file_position *tfp, void *v_state) { example_scanning_state *ess = (example_scanning_state *) v_state; if (tfp->line_count == 1) { match_results mr = Regexp::create_mr(); if ((Regexp::match(&mr, text, L"Example *: *(%**) *(%c+?)")) || (Regexp::match(&mr, text, L"Example *- *(%**) *(%c+?)"))) { ess->star_count = Str::len(mr.exp[0]); if (ess->star_count == 0) { DocumentationCompiler::example_error(ess, I"this example should be marked (before the title) '*', '**', '***' or '****' for difficulty"); ess->star_count = 1; } if (ess->star_count > 4) { DocumentationCompiler::example_error(ess, I"four stars '****' is the maximum difficulty rating allowed"); ess->star_count = 4; } ess->long_title = Str::duplicate(mr.exp[1]); } else { DocumentationCompiler::example_error(ess, I"titling line of example file is malformed"); } Regexp::dispose_of(&mr); } else if (ess->past_header == FALSE) { if (Str::is_whitespace(text)) { ess->past_header = TRUE; return; } match_results mr = Regexp::create_mr(); if (Regexp::match(&mr, text, L"(%C+?) *: *(%c+?)")) { if (Str::eq(mr.exp[0], I"Location")) ess->placement = Str::duplicate(mr.exp[1]); else if (Str::eq(mr.exp[0], I"Description")) ess->desc = Str::duplicate(mr.exp[1]); else { DocumentationCompiler::example_error(ess, I"unknown datum in header line of example file"); } } else { DocumentationCompiler::example_error(ess, I"header line of example file is malformed"); } Regexp::dispose_of(&mr); } else { WRITE_TO(ess->body_text, "%S\n", text); } }
compiled_documentation *DocumentationCompiler::compile_from_file(filename *F, inform_extension *associated_extension) { TEMPORARY_TEXT(temp) TextFiles::read(F, FALSE, "unable to read file of documentation", TRUE, &DocumentationCompiler::read_file_helper, NULL, temp); compiled_documentation *cd = DocumentationCompiler::compile(temp, associated_extension); DISCARD_TEXT(temp) return cd; } void DocumentationCompiler::read_file_helper(text_stream *text, text_file_position *tfp, void *v_state) { text_stream *contents = (text_stream *) v_state; WRITE_TO(contents, "%S\n", text); }
compiled_documentation *DocumentationCompiler::compile(text_stream *source, inform_extension *associated_extension) { SVEXPLAIN(1, "(compiling documentation: %d chars)\n", Str::len(source)); compiled_documentation *cd = DocumentationCompiler::new_wrapper(source); cd->associated_extension = associated_extension; if (cd->associated_extension) WRITE_TO(cd->title, "%X", cd->associated_extension->as_copy->edition->work); if (Str::is_whitespace(source)) cd->empty = TRUE; else Parse the source6.1; if (cd->empty) { SVEXPLAIN(1, "(resulting tree is empty)\n"); } else { SVEXPLAIN(1, "(resulting tree has %d chapter(s), %d section(s) and %d example(s))\n", cd->total_headings[1], cd->total_headings[2], cd->total_examples); } return cd; }
§6.1. The source material is line-based, with semantic content sometimes spreading across multiple lines, so we'll need to keep track of some state as we read one line at a time:
Parse the source6.1 =
int chapter_number = 0, section_number = 0; tree_node *current_headings[4]; Most recent headings of levels 0, 1, 2 current_headings[0] = cd->tree->root; This will never change current_headings[1] = NULL; Latest chapter, if any current_headings[2] = NULL; Latest section in most recent thing of lower level current_headings[3] = NULL; Latest example in most recent thing of lower level tree_node *current_passage = NULL, passage being assembled, if any *current_phrase_defn = NULL, again, if any *current_paragraph = NULL, *current_code = NULL, *last_paste_code = NULL; last code sample with a paste button int pending_code_sample_blanks = 0, code_is_tabular = FALSE; used only when assembling code samples programming_language *default_language = DocumentationCompiler::get_language(I"Inform"); programming_language *language = default_language; Parse the source linewise6.1.1;
- This code is used in §6.
§6.1.1. Leading space on a line is removed but not ignored: it is converted into an indentation level, measured as a tab count, using the exchange rate 4 spaces to 1 tab.
Parse the source linewise6.1.1 =
TEMPORARY_TEXT(line) int indentation = 0, space_count = 0; for (int i=0; i<Str::len(source); i++) { wchar_t c = Str::get_at(source, i); if (c == '\n') { Line read6.1.1.1; Str::clear(line); indentation = 0; space_count = 0; } else if ((Str::len(line) == 0) && (Characters::is_whitespace(c))) { if (c == '\t') indentation++; if (c == ' ') space_count++; if (space_count == 4) { indentation++; space_count = 0; } } else { PUT_TO(line, c); } } if (Str::len(line) > 0) Line read6.1.1.1; Check for runaway phrase definitions6.1.1.2; Complete passage if in one6.1.1.3; DISCARD_TEXT(line)
- This code is used in §6.1.
§6.1.1.1. Trailing space is ignored and removed.
Lines which are unindented and take the following shapes are headings:
Chapter: Survey and Prospecting Section: Black Gold Example: *** Gelignite Anderson - A Tale of the Texas Oilmen
where in each case the colon can equally be a hyphen, and with optional space either side.
Otherwise, lines are divided into blanks, which always end paragraphs but may either end or continue code samples; unindented lines, which are always part of paragraphs; or indented ones, which are always part of code samples.
Line read6.1.1.1 =
Str::trim_white_space(line); if (Str::len(line) == 0) { if (current_paragraph) Complete paragraph or code6.1.1.1.7; if (current_code) Insert line break in code6.1.1.1.12; } else if (indentation == 0) { match_results mr = Regexp::create_mr(); if ((Regexp::match(&mr, line, L"Section *: *(%c+?)")) || (Regexp::match(&mr, line, L"Section *- *(%c+?)"))) { Insert a section heading6.1.1.1.2; } else if ((Regexp::match(&mr, line, L"Chapter *: *(%c+?)")) || (Regexp::match(&mr, line, L"Chapter *- *(%c+?)"))) { Insert a chapter heading6.1.1.1.1; } else if ((Regexp::match(&mr, line, L"Example *: *(%**) *(%c+?)")) || (Regexp::match(&mr, line, L"Example *- *(%**) *(%c+?)"))) { Insert an example heading6.1.1.1.3; } else if (Regexp::match(&mr, line, L"{defn *(%c*?)} *(%c+)")) { Begin a phrase definition6.1.1.1.4; } else if (Regexp::match(&mr, line, L"{end}")) { End a phrase definition6.1.1.1.5; } else { if (current_paragraph == NULL) Begin paragraph6.1.1.1.8; Insert space in paragraph6.1.1.1.9; Insert line in paragraph6.1.1.1.10; } Regexp::dispose_of(&mr); } else { if (current_code == NULL) { if ((Str::get_at(line, 0) == '{') && (Str::get_at(line, 1) == 'a') && (Str::get_at(line, 2) == 's') && (Str::get_at(line, 3) == ' ') && (Str::get_at(line, Str::len(line)-1) == '}')) { Str::delete_n_characters(line, 4); Str::delete_last_character(line); Str::trim_white_space(line); language = DocumentationCompiler::get_language(line); if (language == NULL) { Complete paragraph or code6.1.1.1.7; Begin passage if not already in one6.1.1.1.6; TEMPORARY_TEXT(err) WRITE_TO(err, "cannot find a language called '%S'", line); tree_node *E = DocumentationTree::new_source_error(cd->tree, err); Trees::make_child(E, current_passage); DISCARD_TEXT(err) } } else { Begin code6.1.1.1.11; Insert line in code sample6.1.1.1.13; } } else { Insert line in code sample6.1.1.1.13; } }
- This code is used in §6.1.1 (twice).
§6.1.1.1.1. Insert a chapter heading6.1.1.1.1 =
Complete passage if in one6.1.1.3; chapter_number++; section_number = 0; int level = 1, id = cd->total_headings[0] + cd->total_headings[1] + cd->total_headings[2]; cd->total_headings[1]++; tree_node *new_node = DocumentationTree::new_heading(cd->tree, mr.exp[0], mr.exp[0], level, id, chapter_number, section_number); Place this new structural node in the tree6.1.1.1.1.1;
- This code is used in §6.1.1.1.
§6.1.1.1.2. Insert a section heading6.1.1.1.2 =
Complete passage if in one6.1.1.3; section_number++; int level = 2, id = cd->total_headings[0] + cd->total_headings[1] + cd->total_headings[2]; cd->total_headings[2]++; tree_node *new_node = DocumentationTree::new_heading(cd->tree, mr.exp[0], mr.exp[0], level, id, chapter_number, section_number); Place this new structural node in the tree6.1.1.1.1.1;
- This code is used in §6.1.1.1.
§6.1.1.1.3. Insert an example heading6.1.1.1.3 =
Complete passage if in one6.1.1.3; int level = 3, star_count = Str::len(mr.exp[0]); tree_node *new_node = DocumentationTree::new_example(cd->tree, mr.exp[1], NULL, star_count, ++(cd->total_examples)); Place this new structural node in the tree6.1.1.1.1.1; if (star_count == 0) { Begin passage if not already in one6.1.1.1.6; tree_node *E = DocumentationTree::new_source_error(cd->tree, I"this example should be marked (before the title) '*', '**', '***' or '****' for difficulty"); Trees::make_child(E, current_passage); } if (star_count > 4) { Begin passage if not already in one6.1.1.1.6; tree_node *E = DocumentationTree::new_source_error(cd->tree, I"four stars '****' is the maximum difficulty rating allowed"); Trees::make_child(E, current_passage); }
- This code is used in §6.1.1.1.
§6.1.1.1.4. Begin a phrase definition6.1.1.1.4 =
Check for runaway phrase definitions6.1.1.2; if (current_phrase_defn == NULL) { Begin passage if not already in one6.1.1.1.6; Complete paragraph or code6.1.1.1.7; current_phrase_defn = DocumentationTree::new_phrase_defn(cd->tree, mr.exp[0], mr.exp[1]); Trees::make_child(current_phrase_defn, current_passage); Complete passage if in one6.1.1.3; }
- This code is used in §6.1.1.1.
§6.1.1.1.5. End a phrase definition6.1.1.1.5 =
if (current_phrase_defn) { Complete passage if in one6.1.1.3; current_passage = current_phrase_defn->parent; current_phrase_defn = NULL; } else { Begin passage if not already in one6.1.1.1.6; tree_node *E = DocumentationTree::new_source_error(cd->tree, I"{end} without {defn}"); Trees::make_child(E, current_passage); } language = default_language;
- This code is used in §6.1.1.1.
§6.1.1.1.1.1. Place this new structural node in the tree6.1.1.1.1.1 =
Check for runaway phrase definitions6.1.1.2; for (int j=level-1; j>=0; j--) if (current_headings[j]) { Trees::make_child(new_node, current_headings[j]); break; } current_headings[level] = new_node; for (int j=level+1; j<4; j++) current_headings[j] = NULL; language = default_language;
- This code is used in §6.1.1.1.1, §6.1.1.1.2, §6.1.1.1.3.
§6.1.1.2. Check for runaway phrase definitions6.1.1.2 =
if (current_phrase_defn) { Begin passage if not already in one6.1.1.1.6; tree_node *E = DocumentationTree::new_source_error(cd->tree, I"{defn} has no {end}"); Trees::make_child(E, current_passage); current_phrase_defn = NULL; }
- This code is used in §6.1.1, §6.1.1.1.4, §6.1.1.1.1.1.
§6.1.1.1.6. Begin passage if not already in one6.1.1.1.6 =
if (current_passage == NULL) { current_passage = DocumentationTree::new_passage(cd->tree); if (current_phrase_defn) Trees::make_child(current_passage, current_phrase_defn); else for (int j=3; j>=0; j--) if (current_headings[j]) { Trees::make_child(current_passage, current_headings[j]); break; } current_paragraph = NULL; }
- This code is used in §6.1.1.1, §6.1.1.1.3 (twice), §6.1.1.1.4, §6.1.1.1.5, §6.1.1.2, §6.1.1.1.8, §6.1.1.1.11.
§6.1.1.3. Complete passage if in one6.1.1.3 =
if (current_passage) { Complete paragraph or code6.1.1.1.7; current_passage = NULL; }
- This code is used in §6.1.1, §6.1.1.1.1, §6.1.1.1.2, §6.1.1.1.3, §6.1.1.1.4, §6.1.1.1.5.
§6.1.1.1.7. Complete paragraph or code6.1.1.1.7 =
if (current_paragraph) Complete paragraph6.1.1.1.7.1 if (current_code) Complete code6.1.1.1.7.2
- This code is used in §6.1.1.1 (twice), §6.1.1.1.4, §6.1.1.3, §6.1.1.1.8, §6.1.1.1.11.
§6.1.1.1.8. Line breaks are treated as spaces in the content of a paragraph, so that P->content here can be a long text but one which contains no line breaks.
Begin paragraph6.1.1.1.8 =
Complete paragraph or code6.1.1.1.7; Begin passage if not already in one6.1.1.1.6; current_paragraph = DocumentationTree::new_paragraph(cd->tree, NULL); Trees::make_child(current_paragraph, current_passage);
- This code is used in §6.1.1.1.
§6.1.1.1.9. Insert space in paragraph6.1.1.1.9 =
cdoc_paragraph *P = RETRIEVE_POINTER_cdoc_paragraph(current_paragraph->content); if (Str::len(P->content) > 0) WRITE_TO(P->content, " ");
- This code is used in §6.1.1.1.
§6.1.1.1.10. Insert line in paragraph6.1.1.1.10 =
cdoc_paragraph *P = RETRIEVE_POINTER_cdoc_paragraph(current_paragraph->content); WRITE_TO(P->content, "%S", line);
- This code is used in §6.1.1.1.
§6.1.1.1.7.1. Complete paragraph6.1.1.1.7.1 =
if (current_paragraph) { current_paragraph = NULL; }
- This code is used in §6.1.1.1.7.
§6.1.1.1.11. Line breaks are more significant in code samples, of course. Blank lines at the end of a code sample are stripped out; and they cannot appear at the start of a code sample either, since a non-blank indented line is needed to trigger one.
Begin code6.1.1.1.11 =
Complete paragraph or code6.1.1.1.7; Begin passage if not already in one6.1.1.1.6; int paste_me = FALSE, continue_me = FALSE; if ((Str::get_at(line, 0) == '*') && (Str::get_at(line, 1) == ':')) { Str::delete_first_character(line); Str::delete_first_character(line); Str::trim_white_space(line); paste_me = TRUE; } else if ((Str::get_at(line, 0) == '*') && (Str::get_at(line, 1) == '*') && (Str::get_at(line, 2) == ':')) { Str::delete_first_character(line); Str::delete_first_character(line); Str::delete_first_character(line); Str::trim_white_space(line); continue_me = TRUE; } else if ((Str::get_at(line, 0) == '{') && (Str::get_at(line, 1) == '*') && (Str::get_at(line, 2) == '}')) { Str::delete_first_character(line); Str::delete_first_character(line); Str::delete_first_character(line); Str::trim_white_space(line); paste_me = TRUE; } else if ((Str::get_at(line, 0) == '{') && (Str::get_at(line, 1) == '*') && (Str::get_at(line, 2) == '*') && (Str::get_at(line, 3) == '}')) { Str::delete_first_character(line); Str::delete_first_character(line); Str::delete_first_character(line); Str::delete_first_character(line); Str::trim_white_space(line); continue_me = TRUE; } current_code = DocumentationTree::new_code_sample(cd->tree, paste_me, language); if (continue_me) { if (last_paste_code) { cdoc_code_sample *S = RETRIEVE_POINTER_cdoc_code_sample(last_paste_code->content); S->continuation = current_code; } else { tree_node *E = DocumentationTree::new_source_error(cd->tree, I"cannot continue a paste here"); Trees::make_child(E, current_passage); } } if ((paste_me) || (continue_me)) { last_paste_code = current_code; } else { last_paste_code = NULL; } Trees::make_child(current_code, current_passage); pending_code_sample_blanks = 0; code_is_tabular = FALSE;
- This code is used in §6.1.1.1.
§6.1.1.1.12. Insert line break in code6.1.1.1.12 =
if (current_code->child) { pending_code_sample_blanks++; code_is_tabular = FALSE; }
- This code is used in §6.1.1.1.
§6.1.1.1.13. Insert line in code sample6.1.1.1.13 =
for (int i=0; i<pending_code_sample_blanks; i++) Trees::make_child(DocumentationTree::new_code_line(cd->tree, NULL, 0, FALSE), current_code); pending_code_sample_blanks = 0; match_results mr = Regexp::create_mr(); if (Regexp::match(&mr, line, L"Table %c*")) { code_is_tabular = TRUE; } Regexp::dispose_of(&mr); Trees::make_child(DocumentationTree::new_code_line(cd->tree, line, indentation-1, code_is_tabular), current_code);
- This code is used in §6.1.1.1 (twice).
§6.1.1.1.7.2. Complete code6.1.1.1.7.2 =
if (current_code) { current_code = NULL; code_is_tabular = FALSE; }
- This code is used in §6.1.1.1.7.
programming_language *DocumentationCompiler::get_language(text_stream *name) { inbuild_nest *N = Supervisor::internal(); if (N == NULL) return NULL; pathname *LP = Pathnames::down(Nests::get_location(N), I"PLs"); Languages::set_default_directory(LP); programming_language *pl = Languages::find_by_name(name, NULL, FALSE); if (pl == NULL) LOG("Did not load %S!\n", name); return pl; }