[Core] Refactor the list handling code

The initial implementation was requiring list elements to be in a block
all by themselves, but also indented. This diverged from upstream SAM,
but also made it more complicated to handle the case, as when one drafts
a section, when a section is contained entirely of list elements. The
parser was handling the section block as a field definition followed by
a list.

build.gradle

@@ -7,7 +7,7 @@ plugins {
 }
 
 group 'net.signbit.samx'
-version '0.3.8'
+version '0.4.0'
 
 sourceCompatibility = 1.8
 

doc/samx_language.samx

@@ -52,31 +52,31 @@ chapter: Document Structure
    A SAMx document is comprised of blocks of text. Blocks of text can
    be\:
 
-      * paragraphs
+   * paragraphs
 
-      * lists, either ordered or unordered
+   * lists, either ordered or unordered
 
-      * tabular data
+   * tabular data
 
-      * field definitions
+   * field definitions
 
-      * structured records
+   * structured records
 
-      * definition or references to fragments
+   * definition or references to fragments
 
-      * references to external resources
+   * references to external resources
 
-      * typed (or named blocks)
+   * typed (or named blocks)
 
-      * verbatim code blocks
+   * verbatim code blocks
 
 chapter: SAMx Tools
 
    The following tools are part of the SAMx distribution\:
 
-      * pretty_print
+   * pretty_print
 
-      * to_xml
+   * to_xml
 
    section: pretty_print
 

src/main/antlr/SamXParser.g4

@@ -225,11 +225,11 @@ preciseGridRow : preciseGridRowData | preciseRecordSep ;
 
 externalCode : EXTCODE ;
 
-listElement : condition? flow NEWLINE skipped=NEWLINE? ( INDENT (paragraph | NEWLINE)+ DEDENT )? ( NEWLINE (unorderedList | orderedList) ) ?;
+listElement : condition? flow NEWLINE (separator=NEWLINE? INDENT block+ DEDENT)? ;
 
-unorderedList : INDENT ((BULLET listElement) | NEWLINE)+ DEDENT ;
+unorderedList : (BULLET listElement) NEWLINE* ((BULLET listElement) | NEWLINE)* ;
 
-orderedList : INDENT ((HASH listElement) | NEWLINE)+ DEDENT ;
+orderedList : (HASH listElement) NEWLINE* ((HASH listElement) | NEWLINE)* ;
 
 block :
      NAME TYPESEP attribute* condition? description=flow? NEWLINE+ INDENT block+ DEDENT         # TypedBlock

src/main/java/net/signbit/samx/PrettyPrinterVisitor.java

@@ -78,27 +78,38 @@ private StringBuilder visitParagraphDirect(SamXParser.ParagraphContext ctx)
       return builder;
    }
 
-   @Override
-   public StringBuilder visitParagraph(SamXParser.ParagraphContext ctx)
+   private StringBuilder wrapText(String rawText, int wrapLength, boolean indentFirst)
    {
       StringBuilder builder = new StringBuilder();
-
-      final int wrapLength = wrapParagraphAtColumn - indentLevel * indentString.length();
-
-      final String paragraphText = WordUtils.wrap(visitParagraphDirect(ctx).toString(), wrapLength);
+      final String paragraphText = WordUtils.wrap(rawText, wrapLength);
       StringTokenizer tokenizer = new StringTokenizer(paragraphText, '\n');
       while (tokenizer.hasNext())
       {
-         addIndent(builder);
+         if (! indentFirst)
+         {
+            indentFirst = true;
+         }
+         else
+         {
+            addIndent(builder);
+         }
          builder.append(tokenizer.next());
          builder.append('\n');
       }
 
-      builder.append('\n');
-
       return builder;
    }
 
+   @Override
+   public StringBuilder visitParagraph(SamXParser.ParagraphContext ctx)
+   {
+      final String rawText = visitParagraphDirect(ctx).toString();
+
+      final int wrapLength = wrapParagraphAtColumn - indentLevel * indentString.length();
+
+      return wrapText(rawText, wrapLength, true).append('\n');
+   }
+
    @Override
    public StringBuilder visitTypedBlock(SamXParser.TypedBlockContext ctx)
    {
@@ -138,122 +149,85 @@ private StringBuilder visitList(String type, List<SamXParser.ListElementContext>
    {
       StringBuilder builder = new StringBuilder();
 
-      indentLevel ++;
-
       for (SamXParser.ListElementContext lec : elements)
       {
+         addIndent(builder);
          StringBuilder childBuilder = visitListElement(lec);
          if (childBuilder != null)
          {
-            addIndent(builder);
             builder.append(type);
             builder.append(' ');
             builder.append(childBuilder);
          }
       }
 
-      indentLevel--;
-
       return builder;
    }
 
    @Override
    public StringBuilder visitListElement(SamXParser.ListElementContext ctx)
    {
-      final int wrapLength = wrapParagraphAtColumn - indentLevel * indentString.length() - 2;
-
-      StringBuilder builder = new StringBuilder();
+      StringBuilder firstLineFlow = new StringBuilder();
 
+      final SamXParser.ConditionContext cond = ctx.condition();
+      if (cond != null)
       {
-         StringBuilder firstLineBuilder = new StringBuilder();
+         firstLineFlow.append(visit(cond));
+         firstLineFlow.append(' ');
+      }
 
-         final SamXParser.ConditionContext cond = ctx.condition();
-         if (cond != null)
-         {
-            firstLineBuilder.append(visit(cond));
-            firstLineBuilder.append(' ');
-         }
+      firstLineFlow.append(visitFlow(ctx.flow()));
 
-         firstLineBuilder.append(visitFlow(ctx.flow()));
+      ArrayList<SamXParser.BlockContext> blocks = new ArrayList<>(ctx.block());
 
-         if (ctx.skipped == null)
+      boolean joined = false;
+      if (! blocks.isEmpty())
+      {
+         if (ctx.separator == null)
          {
-            /* This means there is no empty line between the first and second lines in the
-             * bulleted list; by the rules of paragraph handling, we have to join them.
-             */
-
-            if (!ctx.paragraph().isEmpty())
+            SamXParser.BlockContext firstBlock = blocks.get(0);
+            if (firstBlock.getChildCount() == 1)
             {
-               firstLineBuilder.append(' ');
-               firstLineBuilder.append(visitParagraphDirect(ctx.paragraph(0)));
-            }
-         }
+               SamXParser.ParagraphContext pc = firstBlock.getChild(SamXParser.ParagraphContext.class, 0);
 
-         final String wrappedFlow = WordUtils.wrap(firstLineBuilder.toString(), wrapLength);
-         StringTokenizer tokenizer = new StringTokenizer(wrappedFlow, '\n');
+               if (pc != null)
+               {
+                  // need to join the first paragraph with the flow
 
-         boolean firstLine = true;
-         while (tokenizer.hasNext())
-         {
-            if (firstLine)
-            {
-               firstLine = false;
-            }
-            else
-            {
-               addIndent(builder);
-               builder.append("  ");
+                  firstLineFlow.append(' ');
+                  firstLineFlow.append(visitParagraphDirect(pc));
+                  joined = true;
+
+                  blocks.remove(0);
+               }
             }
-            builder.append(tokenizer.next());
-            builder.append('\n');
          }
       }
 
-      if (!ctx.paragraph().isEmpty())
-      {
-         boolean mergedFirstLine = false;
-         if (ctx.skipped == null)
-         {
-            mergedFirstLine = true;
-         }
-         builder.append('\n');
-
-         for (SamXParser.ParagraphContext pc : ctx.paragraph())
-         {
-            if (mergedFirstLine)
-            {
-               mergedFirstLine = false;
-               continue;
-            }
-
-            final String paragraph = visitParagraphDirect(pc).toString();
+      StringBuilder builder = new StringBuilder();
 
-            final String wrappedParagraphs = WordUtils.wrap(paragraph, wrapLength);
-            StringTokenizer tokenizer = new StringTokenizer(wrappedParagraphs, '\n');
+      indentLevel ++;
+      final int wrapLength = wrapParagraphAtColumn - indentLevel * indentString.length();
+      builder.append(wrapText(firstLineFlow.toString(), wrapLength, false));
+      indentLevel --;
 
-            while (tokenizer.hasNext())
-            {
-               addIndent(builder);
-               builder.append("  ");
-               builder.append(tokenizer.next());
-               builder.append('\n');
-            }
+      if (! blocks.isEmpty())
+      {
+         if ((ctx.separator != null) || joined)
+         {
             builder.append('\n');
          }
-      }
-      else
-      {
-         builder.append('\n');
-      }
 
-      if (ctx.unorderedList() != null)
-      {
-         builder.append(visitUnorderedList(ctx.unorderedList()));
+         visitNestedBlock(builder, blocks);
       }
 
-      if (ctx.orderedList() != null)
+      final int builderLen = builder.length();
+      if (builderLen > 2)
       {
-         builder.append(visitOrderedList(ctx.orderedList()));
+         if ((builder.charAt(builderLen - 1) != '\n') || (builder.charAt(builderLen - 2) != '\n'))
+         {
+            builder.append('\n');
+         }
       }
 
       return builder;

src/main/java/net/signbit/samx/XmlTextVisitor.java

@@ -386,54 +386,51 @@ private void visitGenericList(String tagType, List<SamXParser.ListElementContext
             append(getParagraphTag());
             append('>');
             visit(lec.flow());
-            if (lec.skipped == null)
+
+            ArrayList<SamXParser.BlockContext> blocks = new ArrayList<>(lec.block());
+            if (! blocks.isEmpty())
             {
-               /* This means there is no empty line between the first and second lines in the
-                * bulleted list; by the rules of paragraph handling, we have to join them.
-                */
-               if (!lec.paragraph().isEmpty())
+               if (lec.separator == null)
                {
-                  append(' ');
-                  visitParagraphContents(lec.paragraph(0));
+                  SamXParser.BlockContext firstBlock = blocks.get(0);
+                  if (firstBlock.getChildCount() == 1)
+                  {
+                     SamXParser.ParagraphContext pc = firstBlock.getChild(SamXParser.ParagraphContext.class, 0);
+
+                     if (pc != null)
+                     {
+                        // need to join the first paragraph with the flow
+
+                        append(' ');
+                        visitParagraphContents(pc);
+                        blocks.remove(0);
+                     }
+                  }
                }
             }
+
             append("</");
             append(getParagraphTag());
             append('>');
 
-            boolean mergedFirstLine = false;
-            if (lec.skipped == null)
+            if (! blocks.isEmpty())
             {
-               mergedFirstLine = true;
-            }
-            for (SamXParser.ParagraphContext pc : lec.paragraph())
-            {
-               if (mergedFirstLine)
-               {
-                  mergedFirstLine = false;
-                  continue;
-               }
-               visit(pc);
+               appendNewline();
             }
 
-            if (lec.unorderedList() != null)
+            indentParagraph = true;
+            indentLevel ++;
+            for (SamXParser.BlockContext bc: blocks)
             {
-               indentLevel++;
-               appendNewline();
-               visitGenericList(getUnorderedListTag(), lec.unorderedList().listElement());
-               indentLevel--;
-               addIndent();
+               visit(bc);
+               //appendNewline();
             }
+            indentLevel --;
 
-            if (lec.orderedList() != null)
+            if (! blocks.isEmpty())
             {
-               indentLevel++;
-               appendNewline();
-               visitGenericList(getOrderedListTag(), lec.orderedList().listElement());
-               indentLevel--;
                addIndent();
             }
-
             append("</");
             append(getListItemTag());
             append('>');

src/test/java/net/signbit/samx/parser/PrettyPrinterTest.java

@@ -43,7 +43,7 @@ private void testAsPretty(String resourceName, String prettifiedResource)
 
       final String prettified = TestUtils.getResourceContents(prettifiedResource);
 
-      assertEquals(prettified, pretty);
+      assertEquals(prettified.trim(), pretty.trim());
    }
 
    @Test

src/test/java/net/signbit/samx/parser/XmlConverterTest.java

@@ -64,6 +64,8 @@ public void testLists()
       testConversion("lists/multi_line.samx", "lists/multi_line.xml");
 
       testConversion("lists/nested_lists.samx", "lists/nested_lists.xml");
+
+      testConversion("wrap/long_lists.samx", "wrap/long_lists.xml");
    }
 
    @Test

src/test/resources/conditions/mixed-pretty.samx

@@ -15,7 +15,7 @@ section:(?radiation not in {detected, present})
 
 The options are
 
-   * (?radiation in {detected, present}) dead
+* (?radiation in {detected, present}) dead
 
-   * (?radiation==absent) alive
+* (?radiation==absent) alive
 

src/test/resources/conditions/mixed.samx

@@ -14,7 +14,7 @@ section:(?radiation not in {detected, present})
 
 The options are
 
-   * (?radiation in {detected, present}) dead
+* (?radiation in {detected, present}) dead
 
-   * (?radiation==absent) alive
+* (?radiation==absent) alive