Change detection on group and item level

core/src/Debugger.ts

@@ -1,10 +1,12 @@
 import Item from './Item';
 import ItemTransformer from './transformer/ItemTransformer';
 import TransformContext from './transformer/TransformContext';
-import StageResult from './debug/StageResult';
+import StageResult, { initialStage } from './debug/StageResult';
 import ColumnAnnotation from './debug/ColumnAnnotation';
 import AnnotatedColumn from './debug/AnnotatedColumn';
 import { detectChanges } from './debug/detectChanges';
+import { asPages } from './debug/Page';
+import ChangeTracker from './debug/ChangeTracker';
 
 export default class Debugger {
   private context: TransformContext;
@@ -16,42 +18,36 @@ export default class Debugger {
     this.transformers = transformers;
     this.context = context;
     this.stageNames = ['Parse Result', ...transformers.map((t) => t.name)];
-    this.stageResultCache = [
-      {
-        schema: inputSchema.map((column) => ({ name: column })),
-        items: inputItems,
-        changes: new Map(),
-        messages: [
-          `Parsed ${inputItems.length === 0 ? 0 : inputItems[inputItems.length - 1].page + 1} pages with ${
-            inputItems.length
-          } items`,
-        ],
-      },
-    ];
+    this.stageResultCache = [initialStage(inputSchema, inputItems)];
   }
 
-  //TODO return MarkedItem ? (removed, added, etc..)?
   stageResults(stageIndex: number): StageResult {
     for (let idx = 0; idx < stageIndex + 1; idx++) {
       if (!this.stageResultCache[idx]) {
         const transformer = this.transformers[idx - 1];
         const previousStageResult: StageResult = this.stageResultCache[idx - 1];
+        const previousItems = previousStageResult.itemsUnpacked();
         const inputSchema = toSimpleSchema(previousStageResult);
         const outputSchema = transformer.schemaTransformer(inputSchema);
-        const itemResult = transformer.transform(this.context, [...previousStageResult.items]);
+        const itemResult = transformer.transform(this.context, [...previousItems]);
 
-        const changes = detectChanges(previousStageResult.items, itemResult.items);
-
-        const stageResult = {
-          descriptor: transformer.descriptor,
-          schema: toAnnotatedSchema(inputSchema, outputSchema),
-          ...itemResult,
-          changes,
-        };
-        if (changes.size > 0) {
-          stageResult.messages.unshift(`Detected ${changes.size} changes`);
+        const changeTracker = new ChangeTracker();
+        detectChanges(changeTracker, previousItems, itemResult.items);
+        const pages = asPages(changeTracker, itemResult.items, transformer.descriptor.debug?.itemMerger);
+        const messages = itemResult.messages;
+        if (changeTracker.changeCount() > 0) {
+          messages.unshift(`Detected ${changeTracker.changeCount()} changes`);
         }
-        this.stageResultCache.push(stageResult);
+
+        this.stageResultCache.push(
+          new StageResult(
+            transformer.descriptor,
+            toAnnotatedSchema(inputSchema, outputSchema),
+            pages,
+            changeTracker,
+            messages,
+          ),
+        );
       }
     }
     return this.stageResultCache[stageIndex];

core/src/TransformDescriptor.ts

@@ -1,4 +1,4 @@
-import type ItemMerger from './support/ItemMerger';
+import type ItemMerger from './debug/ItemMerger';
 
 interface Debug {
   /**

core/src/assert.ts

@@ -4,6 +4,12 @@ export function assert(condition: boolean, message: string) {
   }
 }
 
+export function assertNot(condition: boolean, message: string) {
+  if (condition) {
+    throw new Error(message || 'Assertion failed');
+  }
+}
+
 export function assertDefined<T>(value: T | undefined, message: string): T {
   if (value === null || typeof value === 'undefined') {
     throw new Error(message || 'Assertion failed');

core/src/debug/ChangeIndex.ts

@@ -0,0 +1,78 @@
+import Item from '../Item';
+
+export default interface ChangeIndex {
+  /**
+   * Return the number of tracked changes.
+   */
+  changeCount(): number;
+
+  /**
+   * Returns the change if for the given item if there is any
+   * @param item
+   */
+  change(item: Item): Change | undefined;
+
+  /**
+   * Returs true if the given item has changed
+   * @param item
+   */
+  hasChanged(item: Item): boolean;
+
+  /**
+   * Returns true if there is a change and it's category is ChangeCategory.PLUS
+   * @param item
+   */
+  isPlusChange(item: Item): boolean;
+
+  /**
+   * Returns true if there is a change and it's category is ChangeCategory.NEUTRAL
+   * @param item
+   */
+  isNeutralChange(item: Item): boolean;
+
+  /**
+   * Returns true if there is a change and it's category is ChangeCategory.MINUS
+   * @param item
+   */
+  isMinusChange(item: Item): boolean;
+}
+
+export abstract class Change {
+  constructor(public category: ChangeCategory) {}
+}
+
+// This is merely for coloring different kind of changes
+export enum ChangeCategory {
+  PLUS = 'PLUS',
+  MINUS = 'MINUS',
+  NEUTRAL = 'NEUTRAL',
+}
+
+export class Addition extends Change {
+  constructor() {
+    super(ChangeCategory.PLUS);
+  }
+}
+
+export class Removal extends Change {
+  constructor() {
+    super(ChangeCategory.MINUS);
+  }
+}
+
+export class ContentChange extends Change {
+  constructor() {
+    super(ChangeCategory.NEUTRAL);
+  }
+}
+
+export enum Direction {
+  UP = 'UP',
+  DOWN = 'DOWN',
+}
+
+export class PositionChange extends Change {
+  constructor(public direction: Direction, public amount: number) {
+    super(direction === Direction.UP ? ChangeCategory.PLUS : ChangeCategory.MINUS);
+  }
+}

core/src/debug/ChangeTracker.ts

@@ -0,0 +1,71 @@
+import ChangeIndex, {
+  Change,
+  ChangeCategory,
+  Addition,
+  Removal,
+  PositionChange,
+  ContentChange,
+  Direction,
+} from './ChangeIndex';
+import Item from '../Item';
+import { assertNot, assertDefined } from '../assert';
+
+const ADDITION = new Addition();
+const REMOVAL = new Removal();
+const CONTENT_CHANGE = new ContentChange();
+
+export default class ChangeTracker implements ChangeIndex {
+  private changes: Map<string, Change> = new Map();
+
+  private addChange(item: Item, change: Change) {
+    const uuid = _uuid(item);
+    assertNot(this.changes.has(uuid), `Change for item ${uuid} already defined`);
+    this.changes.set(uuid, change);
+  }
+
+  changeCount(): number {
+    return this.changes.size;
+  }
+
+  trackAddition(item: Item) {
+    this.addChange(item, ADDITION);
+  }
+
+  trackRemoval(item: Item) {
+    this.addChange(item, REMOVAL);
+  }
+
+  trackPositionalChange(item: Item, oldPosition: number, newPosition: number) {
+    const direction = newPosition > oldPosition ? Direction.DOWN : Direction.UP;
+    const amount = Math.abs(newPosition - oldPosition);
+    this.addChange(item, new PositionChange(direction, amount));
+  }
+
+  trackContentChange(item: Item) {
+    this.addChange(item, CONTENT_CHANGE);
+  }
+
+  change(item: Item): Change | undefined {
+    return this.changes.get(_uuid(item));
+  }
+
+  hasChanged(item: Item): boolean {
+    return this.changes.has(_uuid(item));
+  }
+
+  isPlusChange(item: Item): boolean {
+    return this.change(item)?.category === ChangeCategory.PLUS;
+  }
+
+  isNeutralChange(item: Item): boolean {
+    return this.change(item)?.category === ChangeCategory.NEUTRAL;
+  }
+
+  isMinusChange(item: Item): boolean {
+    return this.change(item)?.category === ChangeCategory.MINUS;
+  }
+}
+
+function _uuid(item: Item): string {
+  return assertDefined(item.uuid, 'UUID is not set');
+}

core/src/debug/ItemGroup.ts

@@ -1,4 +1,4 @@
-import type Item from '../Item';
+import type Item from '..//Item';
 
 export default class ItemGroup {
   top: Item;
@@ -12,4 +12,11 @@ export default class ItemGroup {
   hasMany(): boolean {
     return this.elements.length > 0;
   }
+
+  unpacked(): Item[] {
+    if (this.elements.length > 0) {
+      return this.elements;
+    }
+    return [this.top];
+  }
 }

core/src/debug/ItemMerger.ts

@@ -1,3 +1,4 @@
+import type ChangeTracker from './ChangeTracker';
 import type Item from '../Item';
 
 /**
@@ -5,5 +6,5 @@ import type Item from '../Item';
  */
 export default abstract class ItemMerger {
   constructor(public groupKey: string) {}
-  abstract merge(items: Item[]): Item;
+  abstract merge(tracker: ChangeTracker, items: Item[]): Item;
 }

core/src/debug/LineItemMerger.ts

@@ -1,12 +1,13 @@
 import ItemMerger from './ItemMerger';
 import Item from '../Item';
+import ChangeTracker from './ChangeTracker';
 
 export default class LineItemMerger extends ItemMerger {
-  constructor() {
+  constructor(private trackAsNew = false) {
     super('line');
   }
 
-  merge(items: Item[]): Item {
+  merge(tracker: ChangeTracker, items: Item[]): Item {
     const page = items[0].page;
     const line = items[0].data['line'];
     const str = items.map((item) => item.data['str']).join(' ');
@@ -16,7 +17,7 @@ export default class LineItemMerger extends ItemMerger {
     const height = Math.max(...items.map((item) => item.data['height']));
     const fontNames = [...new Set(items.map((item) => item.data['fontName']))];
     const directions = [...new Set(items.map((item) => item.data['dir']))];
-    return new Item(page, {
+    const newItem = new Item(page, {
       str,
       line,
       x,
@@ -26,5 +27,12 @@ export default class LineItemMerger extends ItemMerger {
       fontName: fontNames,
       dir: directions,
     });
+
+    if (this.trackAsNew) {
+      tracker.trackAddition(newItem);
+    } else if (items.find((item) => tracker.hasChanged(item))) {
+      tracker.trackContentChange(newItem);
+    }
+    return newItem;
   }
 }

core/src/debug/Page.ts

@@ -0,0 +1,29 @@
+import Item from '../Item';
+import { groupByElement, groupByPage } from '../support/groupingUtils';
+import ChangeTracker from './ChangeTracker';
+import ItemGroup from './ItemGroup';
+import ItemMerger from './ItemMerger';
+
+export default interface Page {
+  index: number;
+  itemGroups: ItemGroup[];
+}
+
+export function asPages(tracker: ChangeTracker, items: Item[], itemMerger?: ItemMerger): Page[] {
+  return groupByPage(items).map((pageItems: Item[]) => {
+    let itemGroups: ItemGroup[];
+    if (itemMerger) {
+      itemGroups = groupByElement(pageItems, itemMerger.groupKey).map((groupItems) => {
+        if (groupItems.length > 1) {
+          const top = itemMerger.merge(tracker, groupItems);
+          return new ItemGroup(top, groupItems);
+        } else {
+          return new ItemGroup(groupItems[0]);
+        }
+      });
+    } else {
+      itemGroups = pageItems.map((item) => new ItemGroup(item));
+    }
+    return { index: pageItems[0].page, itemGroups } as Page;
+  });
+}

core/src/debug/StageResult.ts

@@ -1,12 +1,35 @@
-import TransformDescriptor from '../TransformDescriptor';
-import Item from '../Item';
+import TransformDescriptor, { toDescriptor } from '../TransformDescriptor';
 import AnnotatedColumn from './AnnotatedColumn';
-import { Change } from './detectChanges';
+import Item from '../Item';
+import Page, { asPages } from './Page';
+import ChangeIndex from './ChangeIndex';
+import ChangeTracker from './ChangeTracker';
 
-export default interface StageResult {
-  descriptor?: TransformDescriptor;
-  schema: AnnotatedColumn[];
-  items: Item[];
-  changes: Map<string, Change>;
-  messages: string[];
+export default class StageResult {
+  constructor(
+    public descriptor: TransformDescriptor,
+    public schema: AnnotatedColumn[],
+    public pages: Page[],
+    public changes: ChangeIndex,
+    public messages: string[],
+  ) {}
+
+  itemsUnpacked(): Item[] {
+    return this.pages.reduce((items: Item[], page: Page) => {
+      page.itemGroups.forEach((itemGroup) => itemGroup.unpacked().forEach((item) => items.push(item)));
+      return items;
+    }, []);
+  }
+}
+
+export function initialStage(inputSchema: string[], inputItems: Item[]): StageResult {
+  const schema = inputSchema.map((column) => ({ name: column }));
+  const tracker = new ChangeTracker();
+  const pages = asPages(tracker, inputItems);
+  const messages = [
+    `Parsed ${inputItems.length === 0 ? 0 : inputItems[inputItems.length - 1].page + 1} pages with ${
+      inputItems.length
+    } items`,
+  ];
+  return new StageResult(toDescriptor({}), schema, pages, tracker, messages);
 }

core/src/debug/detectChanges.ts

@@ -1,67 +1,26 @@
+import ChangeTracker from './ChangeTracker';
 import { assertDefined } from '../assert';
 import Item from '../Item';
 
-// export enum ChangeType {
-//   POSITION_CHANGED = 'POSITION_CHANGED',
-//   ADDED = 'ADDED',
-//   REMOVED = 'REMOVED',
-//   CHANGED = 'CHANGED',
-//   EVALUATED = 'EVALUATED',
-// }
-
-// Marker interface for a change
-export abstract class Change {
-  constructor(public category: ChangeCategory) {}
-}
-
-// This is merely for coloring different kind of changes
-export enum ChangeCategory {
-  PLUS = 'PLUS',
-  MINUS = 'MINUS',
-  NEUTRAL = 'NEUTRAL',
-}
-
-export class Addition extends Change {
-  constructor() {
-    super(ChangeCategory.PLUS);
-  }
-}
-
-export enum Direction {
-  UP = 'UP',
-  DOWN = 'DOWN',
-}
-
-export class PositionChange extends Change {
-  constructor(public direction: Direction, public amount: number) {
-    super(direction === Direction.UP ? ChangeCategory.PLUS : ChangeCategory.NEUTRAL);
-  }
-}
-
 /**
  * Compares incomming and outgoing items of a transformer in order to detect changes and to display them in any debug visualization.
  */
-export function detectChanges(inputItems: Item[], transformedItems: Item[]): Map<string, Change> {
-  const changes: Map<string, Change> = new Map();
+export function detectChanges(tracker: ChangeTracker, inputItems: Item[], transformedItems: Item[]) {
   const inputItemsByUuid = inputMap(inputItems);
   transformedItems.forEach((item, idx) => {
-    const uuid = getUuid(item);
+    const uuid = _uuid(item);
     const oldItem = inputItemsByUuid.get(uuid);
     if (oldItem) {
       if (idx !== oldItem.position) {
-        const direction = idx > oldItem.position ? Direction.DOWN : Direction.UP;
-        const amount = Math.abs(idx - oldItem.position);
-        changes.set(uuid, new PositionChange(direction, amount));
+        tracker.trackPositionalChange(item, oldItem.position, idx);
       }
       //TODO check for change
     } else {
-      changes.set(uuid, new Addition());
+      tracker.trackAddition(item);
     }
   });
 
   //TODO detect removals (need to re-add them ?)
-
-  return changes;
 }
 
 interface InputItem {
@@ -71,11 +30,11 @@ interface InputItem {
 
 function inputMap(inputItems: Item[]): Map<string, InputItem> {
   return inputItems.reduce((map, item, idx) => {
-    map.set(getUuid(item), { item, position: idx });
+    map.set(_uuid(item), { item, position: idx });
     return map;
   }, new Map());
 }
 
-function getUuid(item: Item): string {
+function _uuid(item: Item): string {
   return assertDefined(item.uuid, 'UUID is not set');
 }

core/src/support/Page.ts

@@ -1,6 +0,0 @@
-import type ItemGroup from './ItemGroup';
-
-export default interface Page {
-  index: number;
-  itemGroups: ItemGroup[];
-}

core/src/support/groupingUtils.ts

@@ -1,7 +1,8 @@
-import ItemMerger from './ItemMerger';
+import ItemMerger from '../debug/ItemMerger';
 import Item from '../Item';
-import ItemGroup from './ItemGroup';
-import Page from './Page';
+import ItemGroup from '../debug/ItemGroup';
+import Page from '../debug/Page';
+import ChangeTracker from '../debug/ChangeTracker';
 
 type KeyExtractor = (item: Item) => any;
 type PageItemTransformer = (page: number, items: Item[]) => Item[];
@@ -42,22 +43,3 @@ export function transformGroupedByPageAndLine(items: Item[], groupedTransformer:
   });
   return transformedItems;
 }
-
-export function asPages(items: Item[], itemMerger?: ItemMerger): Page[] {
-  return groupByPage(items).map((pageItems: Item[]) => {
-    let itemGroups: ItemGroup[];
-    if (itemMerger) {
-      itemGroups = groupByElement(pageItems, itemMerger.groupKey).map((groupItems) => {
-        if (groupItems.length > 1) {
-          const top = itemMerger.merge(groupItems);
-          return new ItemGroup(top, groupItems);
-        } else {
-          return new ItemGroup(groupItems[0]);
-        }
-      });
-    } else {
-      itemGroups = pageItems.map((item) => new ItemGroup(item));
-    }
-    return { index: pageItems[0].page, itemGroups } as Page;
-  });
-}

core/src/transformer/AdjustHeight.ts

@@ -1,9 +1,8 @@
-import PageViewport from '../parse/PageViewport';
 import Item from '../Item';
 import ItemResult from '../ItemResult';
 import ItemTransformer from './ItemTransformer';
 import TransformContext from './TransformContext';
-import { transformGroupedByPage } from '../support/itemUtils';
+import { transformGroupedByPage } from '../support/groupingUtils';
 
 export default class AdjustHeight extends ItemTransformer {
   constructor() {

core/src/transformer/CompactLines.ts

@@ -2,8 +2,8 @@ import Item from '../Item';
 import ItemResult from '../ItemResult';
 import ItemTransformer from './ItemTransformer';
 import TransformContext from './TransformContext';
-import { transformGroupedByPage } from '../support/itemUtils';
-import LineItemMerger from '../support/LineItemMerger';
+import LineItemMerger from '../debug/LineItemMerger';
+import { transformGroupedByPage } from '../support/groupingUtils';
 
 export default class CompactLines extends ItemTransformer {
   constructor() {
@@ -13,7 +13,7 @@ export default class CompactLines extends ItemTransformer {
       {
         requireColumns: ['str', 'y', 'height'],
         debug: {
-          itemMerger: new LineItemMerger(),
+          itemMerger: new LineItemMerger(true),
         },
       },
       (incomingSchema) => {

core/src/transformer/SortXWithinLines.ts

@@ -2,8 +2,8 @@ import Item from '../Item';
 import ItemResult from '../ItemResult';
 import ItemTransformer from './ItemTransformer';
 import TransformContext from './TransformContext';
-import LineItemMerger from '../support/LineItemMerger';
-import { transformGroupedByPageAndLine } from '../support/itemUtils';
+import LineItemMerger from '../debug/LineItemMerger';
+import { transformGroupedByPageAndLine } from '../support/groupingUtils';
 
 export default class SortXWithinLines extends ItemTransformer {
   constructor() {

core/test/Debugger.test.ts

@@ -3,17 +3,19 @@ import Item from 'src/Item';
 import ItemTransformer from 'src/transformer/ItemTransformer';
 import TransformDescriptor from 'src/TransformDescriptor';
 import TransformContext from 'src/transformer/TransformContext';
+import LineItemMerger from 'src/debug/LineItemMerger';
 import ItemResult from 'src/ItemResult';
 import ColumnAnnotation from 'src/debug/ColumnAnnotation';
 import AnnotatedColumn from 'src/debug/AnnotatedColumn';
+import { items } from './testItems';
 
 class TestTransformer extends ItemTransformer {
   items: Item[];
   constructor(name: string, descriptor: Partial<TransformDescriptor>, outputSchema: string[], items: Item[]) {
-    super(name, `Description for ${name}`, descriptor, (incomingSchema) => outputSchema);
+    super(name, `Description for ${name}`, descriptor, (_) => outputSchema);
     this.items = items;
   }
-  transform(_: TransformContext, items: Item[]): ItemResult {
+  transform(_: TransformContext, __: Item[]): ItemResult {
     return {
       items: this.items,
       messages: [],
@@ -21,9 +23,13 @@ class TestTransformer extends ItemTransformer {
   }
 }
 
-test('basic debug', async () => {
+describe('Transform Items', () => {
+  test('Basics', async () => {
     const parsedSchema = ['A', 'B'];
-  const parsedItems = [new Item(0, { A: 'a_row1', B: 'b_row1' }), new Item(0, { A: 'a_row2', B: 'b_row2' })];
+    const parsedItems = items(0, [
+      { A: 'a_row1', B: 'b_row1' },
+      { A: 'a_row2', B: 'b_row2' },
+    ]);
 
     const trans1Desc = { requireColumns: ['A', 'B'] };
     const trans1Schema = ['C'];
@@ -38,10 +44,91 @@ test('basic debug', async () => {
       ...parsedSchema.map((column) => ({ name: column, annotation: ColumnAnnotation.REMOVED })),
       { name: 'C', annotation: ColumnAnnotation.ADDED },
     ]);
-  expect(debug.stageResults(0).items).toEqual(parsedItems);
-  expect(debug.stageResults(1).items).toEqual(trans1Items);
+
+    expect(debug.stageResults(0).itemsUnpacked()).toEqual(parsedItems);
+    expect(debug.stageResults(1).itemsUnpacked()).toEqual(trans1Items);
   });
 
+  test('Line Merge', async () => {
+    const parsedSchema = ['id', 'y'];
+    const parsedItems = items(0, [
+      { id: 1, y: 1 },
+      { id: 2, y: 2 },
+      { id: 3, y: 2 },
+    ]);
+
+    const trans1Desc = { requireColumns: ['id', 'y'], debug: { itemMerger: new LineItemMerger(true) } };
+    const trans1Schema = ['id', 'line'];
+    const trans1Items = parsedItems.map((item) => item.withData({ line: item.data['y'] }));
+
+    const transformers = [new TestTransformer('Trans1', trans1Desc, trans1Schema, trans1Items)];
+    const debug = new Debugger(parsedSchema, parsedItems, { fontMap: new Map(), pageViewports: [] }, transformers);
+
+    expect(debug.stageNames).toEqual(['Parse Result', 'Trans1']);
+    expect(debug.stageResults(0).schema).toEqual([{ name: 'id' }, { name: 'y' }]);
+    expect(debug.stageResults(1).schema).toEqual([
+      { name: 'id' },
+      { name: 'y', annotation: ColumnAnnotation.REMOVED },
+      { name: 'line', annotation: ColumnAnnotation.ADDED },
+    ]);
+
+    expect(debug.stageResults(0).itemsUnpacked()).toEqual(parsedItems);
+    expect(debug.stageResults(1).itemsUnpacked()).toEqual(trans1Items);
+
+    const lineMergingStage = debug.stageResults(1);
+    const { changes, pages } = lineMergingStage;
+
+    //verify item groups
+    expect(pages[0].itemGroups.map((itemGroup) => changes.hasChanged(itemGroup.top))).toEqual([false, true]);
+
+    //verify unpacked items
+    expect(lineMergingStage.itemsUnpacked().map((item) => changes.hasChanged(item))).toEqual([false, false, false]);
+  });
+});
+
+test('Change inside of Line', async () => {
+  const parsedSchema = ['id', 'line'];
+  const parsedItems = items(0, [
+    { id: 1, line: 1 },
+    { id: 2, line: 1 },
+    { id: 3, line: 2 },
+    { id: 4, line: 2 },
+  ]);
+
+  const trans1Desc = { requireColumns: ['id', 'line'], debug: { itemMerger: new LineItemMerger() } };
+  const trans1Schema = parsedSchema;
+  const trans1Items = swapElements([...parsedItems], 0, 1);
+
+  const transformers = [new TestTransformer('Trans1', trans1Desc, trans1Schema, trans1Items)];
+  const debug = new Debugger(parsedSchema, parsedItems, { fontMap: new Map(), pageViewports: [] }, transformers);
+
+  expect(debug.stageNames).toEqual(['Parse Result', 'Trans1']);
+  expect(debug.stageResults(0).schema).toEqual([{ name: 'id' }, { name: 'line' }]);
+  expect(debug.stageResults(1).schema).toEqual([{ name: 'id' }, { name: 'line' }]);
+  expect(debug.stageResults(0).itemsUnpacked()).toEqual(parsedItems);
+  expect(debug.stageResults(1).itemsUnpacked()).toEqual(trans1Items);
+
+  const { changes, pages } = debug.stageResults(1);
+
+  //verify item groups
+  expect(pages[0].itemGroups.map((itemGroup) => changes.hasChanged(itemGroup.top))).toEqual([true, false]);
+
+  //verify unpacked items
+  expect(
+    debug
+      .stageResults(1)
+      .itemsUnpacked()
+      .map((item) => changes.hasChanged(item)),
+  ).toEqual([true, true, false, false]);
+});
+
+var swapElements = function (arr: Item[], indexA: number, indexB: number): Item[] {
+  var temp = arr[indexA];
+  arr[indexA] = arr[indexB];
+  arr[indexB] = temp;
+  return arr;
+};
+
 describe('build schemas', () => {
   const items: Item[] = [];
 

core/test/debug/LineItemMerger.test.ts

@@ -1,13 +1,15 @@
 import LineItemMerger from 'src/debug/LineItemMerger';
+import ChangeTracker from 'src/debug/ChangeTracker';
 import Item from 'src/Item';
-import { items } from '../testItems';
-
-const itemMerger = new LineItemMerger();
+import { items, realisticItems } from '../testItems';
+import { Addition, ContentChange } from 'src/debug/ChangeIndex';
 
 test('Basics', async () => {
+  const itemMerger = new LineItemMerger();
+  const tracker = new ChangeTracker();
   expect(itemMerger.groupKey).toEqual('line');
-
-  const mergedItem = itemMerger?.merge(
+  const mergedItem = itemMerger.merge(
+    tracker,
     items(0, [
       {
         line: 2,
@@ -41,7 +43,7 @@ test('Basics', async () => {
       },
     ]),
   );
-  expect(mergedItem?.withoutUuid()).toEqual(
+  expect(mergedItem.withoutUuid()).toEqual(
     new Item(0, {
       line: 2,
       x: 240,
@@ -54,3 +56,22 @@ test('Basics', async () => {
     }).withoutUuid(),
   );
 });
+
+test('Track all lines as changes', async () => {
+  const itemMerger = new LineItemMerger(true);
+  const tracker = new ChangeTracker();
+  const mergedItem = itemMerger.merge(tracker, realisticItems(0, [{ line: 1 }, { line: 1 }]));
+  expect(tracker.change(mergedItem)).toEqual(new Addition());
+});
+
+test('Mark lines containing changed items as changed', async () => {
+  const itemMerger = new LineItemMerger();
+  const tracker = new ChangeTracker();
+  const items1 = realisticItems(0, [{ line: 1 }, { line: 1 }]);
+  const items2 = realisticItems(0, [{ line: 2 }, { line: 2 }]);
+  tracker.trackPositionalChange(items1[1], 1, 0);
+  const mergedItem1 = itemMerger.merge(tracker, items1);
+  const mergedItem2 = itemMerger.merge(tracker, items2);
+  expect(tracker.change(mergedItem1)).toEqual(new ContentChange());
+  expect(tracker.change(mergedItem2)).toEqual(undefined);
+});

core/test/debug/Page.test.ts

@@ -0,0 +1,60 @@
+import Item from 'src/Item';
+import Page, { asPages } from 'src/debug/Page';
+import ItemGroup from 'src/debug/ItemGroup';
+import ItemMerger from 'src/debug/ItemMerger';
+import { items } from 'test/testItems';
+import ChangeTracker from 'src/debug/ChangeTracker';
+
+test('empty', async () => {
+  const tracker = new ChangeTracker();
+  expect(asPages(tracker, [])).toEqual([]);
+});
+
+test('no merger', async () => {
+  const pageItems = [
+    items(0, [{ id: 1, line: 1 }]),
+    items(1, [
+      { id: 2, line: 1 },
+      { id: 3, line: 1 },
+      { id: 4, line: 1 },
+    ]),
+    items(2, [{ id: 5, line: 1 }]),
+  ];
+  const flattenedItems = new Array<Item>().concat(...pageItems);
+  const tracker = new ChangeTracker();
+  const pages = asPages(tracker, flattenedItems);
+  expect(pages).toEqual([
+    { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) },
+    { index: 1, itemGroups: pageItems[1].map((item) => new ItemGroup(item)) },
+    { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) },
+  ] as Page[]);
+  expect(tracker.changeCount()).toEqual(0);
+});
+
+test('merger', async () => {
+  const pageItems = [
+    items(0, [{ id: 1, line: 1 }]),
+    items(1, [
+      { id: 2, line: 1 },
+      { id: 3, line: 1 },
+      { id: 4, line: 2 },
+    ]),
+    items(2, [{ id: 5, line: 1 }]),
+  ];
+  const flattenedItems = new Array<Item>().concat(...pageItems);
+  const merger: ItemMerger = { groupKey: 'line', merge: (items) => items[0] };
+  const tracker = new ChangeTracker();
+  const pages = asPages(tracker, flattenedItems, merger);
+
+  expect(pages).toEqual([
+    { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) },
+    {
+      index: 1,
+      itemGroups: [
+        new ItemGroup(merger.merge(tracker, pageItems[1].slice(0, 2)), pageItems[1].slice(0, 2)),
+        new ItemGroup(pageItems[1][2]),
+      ],
+    },
+    { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) },
+  ] as Page[]);
+});

core/test/debug/detectChanges.test.ts

@@ -1,4 +1,6 @@
-import { detectChanges, PositionChange, Direction } from 'src/debug/detectChanges';
+import ChangeTracker from 'src/debug/ChangeTracker';
+import { detectChanges } from 'src/debug/detectChanges';
+import { PositionChange, Direction } from 'src/debug/ChangeIndex';
 import { items } from 'test/testItems';
 
 test('No changes', async () => {
@@ -9,8 +11,9 @@ test('No changes', async () => {
     { id: 'B', line: '1', x: 2 },
   ]);
 
-  const changes = detectChanges(inputItems, inputItems);
-  expect(changes).toEqual(new Map());
+  const tracker = new ChangeTracker();
+  detectChanges(tracker, inputItems, inputItems);
+  expect(tracker.changeCount()).toEqual(0);
 });
 test('Positional changes', async () => {
   const inputItems = items(0, [
@@ -22,11 +25,9 @@ test('Positional changes', async () => {
 
   const transformedItems = [inputItems[0], inputItems[3], inputItems[2], inputItems[1]];
 
-  const changes = detectChanges(inputItems, transformedItems);
-  expect(changes).toEqual(
-    new Map([
-      [inputItems[1].uuid, new PositionChange(Direction.DOWN, 2)],
-      [inputItems[3].uuid, new PositionChange(Direction.UP, 2)],
-    ]),
-  );
+  const tracker = new ChangeTracker();
+  const changes = detectChanges(tracker, inputItems, transformedItems);
+  expect(tracker.changeCount()).toEqual(2);
+  expect(tracker.change(inputItems[1])).toEqual(new PositionChange(Direction.DOWN, 2));
+  expect(tracker.change(inputItems[3])).toEqual(new PositionChange(Direction.UP, 2));
 });

core/test/support/groupingUtils.test.ts

@@ -1,14 +1,10 @@
 import Item from 'src/Item';
-import Page from 'src/support/Page';
 import {
   groupByPage,
   groupByElement,
   transformGroupedByPage,
   transformGroupedByPageAndLine,
-  asPages,
-} from 'src/support/itemUtils';
-import ItemGroup from 'src/support/ItemGroup';
-import ItemMerger from 'src/support/ItemMerger';
+} from 'src/support/groupingUtils';
 import { items } from 'test/testItems';
 
 describe('groupByPage', () => {
@@ -97,46 +93,3 @@ describe('transformGroupedByPageAndLine', () => {
     expect(transformedItems.map((item) => item.data['group'])).toEqual(['0/1:1', '1/1:2', '1/2:1', '2/1:1']);
   });
 });
-
-describe('asPages', () => {
-  test('empty', async () => {
-    expect(groupByPage([])).toEqual([]);
-  });
-
-  test('no merger', async () => {
-    const pageItems = [
-      [new Item(0, { id: 1, line: 1 })],
-      [new Item(1, { id: 2, line: 1 }), new Item(1, { id: 3, line: 1 }), new Item(1, { id: 4, line: 2 })],
-      [new Item(2, { id: 5, line: 1 })],
-    ];
-    const flattenedItems = new Array<Item>().concat(...pageItems);
-    const pages = asPages(flattenedItems);
-    expect(pages).toEqual([
-      { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) },
-      { index: 1, itemGroups: pageItems[1].map((item) => new ItemGroup(item)) },
-      { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) },
-    ] as Page[]);
-  });
-
-  test('merger', async () => {
-    const pageItems = [
-      [new Item(0, { id: 1, line: 1 })],
-      [new Item(1, { id: 2, line: 1 }), new Item(1, { id: 3, line: 1 }), new Item(1, { id: 4, line: 2 })],
-      [new Item(2, { id: 5, line: 1 })],
-    ];
-    const flattenedItems = new Array<Item>().concat(...pageItems);
-    const merger: ItemMerger = { groupKey: 'line', merge: (items) => items[0] };
-    const pages = asPages(flattenedItems, merger);
-    expect(pages).toEqual([
-      { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) },
-      {
-        index: 1,
-        itemGroups: [
-          new ItemGroup(merger.merge(pageItems[1].slice(0, 2)), pageItems[1].slice(0, 2)),
-          new ItemGroup(pageItems[1][2]),
-        ],
-      },
-      { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) },
-    ] as Page[]);
-  });
-});

core/test/testItems.ts

@@ -3,3 +3,20 @@ import Item from 'src/Item';
 export function items(page: number, data: object[]): Item[] {
   return data.map((data) => new Item(page, data));
 }
+
+export function realisticItems(page: number, data: object[]): Item[] {
+  return data.map(
+    (data, idx) =>
+      new Item(page, {
+        line: idx,
+        x: idx,
+        y: idx,
+        str: `text ${idx}`,
+        fontName: 'g_d0_f2',
+        dir: 'ltr',
+        width: 10,
+        height: 10,
+        ...data,
+      }),
+  );
+}

ui/src/components/ComponentDefinition.ts

@@ -0,0 +1,5 @@
+import type { SvelteComponent } from 'svelte';
+
+export default class ComponentDefinition {
+    constructor(public component: object, public args: object = {}) {}
+}

ui/src/debug/ChangeSymbol.svelte

@@ -1,25 +1,40 @@
 <script>
-    import type { Change } from '@core/debug/detectChanges';
-    import { PositionChange, Direction } from '../../../core/src/debug/detectChanges';
+    import type ChangeIndex from '@core/debug/ChangeIndex';
+    import type Item from '@core/Item';
+    import { Addition, ContentChange, PositionChange, Direction } from '../../../core/src/debug/ChangeIndex';
+    import ComponentDefinition from '../components/ComponentDefinition';
 
-    import Icon from 'fa-svelte';
-    import type { IconDefinition } from '@fortawesome/fontawesome-common-types/index';
-    import { faArrowUp as up } from '@fortawesome/free-solid-svg-icons/faArrowUp';
-    import { faArrowDown as down } from '@fortawesome/free-solid-svg-icons/faArrowDown';
+    import {
+        PlusCircle as Plus,
+        Adjustments as Changed,
+        ArrowCircleUp as Up,
+        ArrowCircleDown as Down,
+    } from 'svelte-hero-icons';
 
-    export let changes: Map<string, Change>;
-    export let itemUid: string;
+    export let changes: ChangeIndex;
+    export let item: Item;
 
+    $: hasChanged = changes.hasChanged(item);
     let changeContent: string;
-    let icon: IconDefinition;
+    let iconComp: ComponentDefinition;
     $: {
-        const change = changes.get(itemUid);
-        if (change) {
+        if (hasChanged) {
+            let args = { size: '14' };
+            let change = changes.change(item);
             switch (change.constructor.name) {
                 case PositionChange.name:
                     const positionChange = change as PositionChange;
                     changeContent = `${positionChange.amount}`;
-                    icon = positionChange.direction === Direction.UP ? up : down;
+                    iconComp =
+                        positionChange.direction === Direction.UP
+                            ? new ComponentDefinition(Up, args)
+                            : new ComponentDefinition(Down, args);
+                    break;
+                case Addition.name:
+                    iconComp = new ComponentDefinition(Plus, args);
+                    break;
+                case ContentChange.name:
+                    iconComp = new ComponentDefinition(Changed, args);
                     break;
                 default:
                     throw new Error(`${change.constructor.name}: ${change}`);
@@ -28,11 +43,13 @@
     }
 </script>
 
-{#if changeContent}
+{#if hasChanged}
     <div class="flex space-x-0.5 items-center text-xs">
-        {#if icon}
-            <Icon {icon} />
+        {#if iconComp}
+            <svelte:component this={iconComp.component} {...iconComp.args} />
         {/if}
+        {#if changeContent}
             <div>{changeContent}</div>
+        {/if}
     </div>
 {/if}

ui/src/debug/DebugView.svelte

@@ -5,7 +5,6 @@
     import { BookOpen, ArrowLeft, ArrowRight } from 'svelte-hero-icons';
 
     import type Debugger from '@core/Debugger';
-    import { asPages } from '../../../core/src/support/itemUtils';
 
     import Popup from '../components/Popup.svelte';
     import PageSelectionPopup from './PageSelectionPopup.svelte';
@@ -24,10 +23,9 @@
     $: canPrev = currentStage > 0;
     $: stageResult = debug.stageResults(currentStage);
     $: pageIsPinned = !isNaN(pinnedPage);
-    $: pagesNumbers = new Set(stageResult.items.map((item) => item.page));
-    $: pages = asPages(stageResult.items, stageResult.descriptor?.debug?.itemMerger);
+    $: pagesNumbers = new Set(stageResult.pages.map((page) => page.index));
     $: maxPage = Math.max(...pagesNumbers);
-    $: visiblePages = pageIsPinned ? pages.filter((page) => page.index === pinnedPage) : pages;
+    $: visiblePages = pageIsPinned ? stageResult.pages.filter((page) => page.index === pinnedPage) : stageResult.pages;
 </script>
 
 <div class="mx-4">

ui/src/debug/ItemTable.svelte

@@ -2,11 +2,10 @@
     import { scale, fade } from 'svelte/transition';
     import type AnnotatedColumn from '@core/debug/AnnotatedColumn';
     import ColumnAnnotation from '../../../core/src/debug/ColumnAnnotation';
-    import type { Change } from '@core/debug/detectChanges';
-    import { ChangeCategory } from '../../../core/src/debug/detectChanges';
+    import type ChangeIndex from '@core/debug/ChangeIndex';
     import inView from '../actions/inView';
     import { formatValue } from './formatValues';
-    import type Page from '@core/support/Page';
+    import type Page from '@core/debug/Page';
     import ChangeSymbol from './ChangeSymbol.svelte';
 
     export let schema: AnnotatedColumn[];
@@ -14,7 +13,7 @@
     export let maxPage: number;
     export let pageIsPinned: boolean;
     export let onlyRelevantItems: boolean;
-    export let changes: Map<string, Change>;
+    export let changes: ChangeIndex;
     let maxItemsToRenderInOneLoad = 200;
     let renderedMaxPage = 0;
     let expandedItemGroup: { pageIndex: number; itemIndex: number };
@@ -75,17 +74,17 @@
             {/if}
 
             <!-- Page items -->
-            {#each page.itemGroups.filter((group) => !onlyRelevantItems || changes.has(group.top.uuid)) as itemGroup, itemIdx}
+            {#each page.itemGroups.filter((group) => !onlyRelevantItems || changes.hasChanged(group.top)) as itemGroup, itemIdx}
                 <tr
                     class:expandable={itemGroup.hasMany()}
                     class:expanded={expandedItemGroup && isExpanded(page.index, itemIdx)}
-                    class:changePlus={changes.get(itemGroup.top.uuid)?.category === ChangeCategory.PLUS}
-                    class:changeNeutral={changes.get(itemGroup.top.uuid)?.category === ChangeCategory.NEUTRAL}
-                    class:changeMinus={changes.get(itemGroup.top.uuid)?.category === ChangeCategory.MINUS}
+                    class:changePlus={changes.isPlusChange(itemGroup.top)}
+                    class:changeNeutral={changes.isNeutralChange(itemGroup.top)}
+                    class:changeMinus={changes.isMinusChange(itemGroup.top)}
                     in:fade>
                     <!-- Page number in first page item row -->
                     {#if itemIdx === 0}
-                        <td id="page" class="page bg-gray-50">
+                        <td id="page" class="page bg-gray-50 align-top">
                             <div>Page {page.index} {pageIsPinned ? '' : ' / ' + maxPage}</div>
                         </td>
                     {:else}
@@ -93,9 +92,11 @@
                     {/if}
                     <span class="contents" on:click={() => itemGroup.hasMany() && toggleRow(page.index, itemIdx)}>
                         <!-- ID & change marker column -->
-                        <td class="flex space-x-1">
-                            <div>{itemIdx}{itemGroup.hasMany() ? '+' : ''}</div>
-                            <ChangeSymbol {changes} itemUid={itemGroup.top.uuid} />
+                        <td>
+                            <div class="flex space-x-0.5 items-center">
+                                <ChangeSymbol {changes} item={itemGroup.top} />
+                                <div>{itemIdx}{itemGroup.hasMany() ? '…' : ''}</div>
+                            </div>
                         </td>
 
                         <!-- Row values -->
@@ -108,9 +109,18 @@
                 <!-- Expanded childs -->
                 {#if expandedItemGroup && isExpanded(page.index, itemIdx)}
                     {#each itemGroup.elements as child, childIdx}
-                        <tr class="childs">
+                        <tr
+                            class="childs"
+                            class:changePlus={changes.isPlusChange(child)}
+                            class:changeNeutral={changes.isNeutralChange(child)}
+                            class:changeMinus={changes.isMinusChange(child)}>
                             <td id="page" />
-                            <td class="whitespace-nowrap">{'└ ' + childIdx}</td>
+                            <td class="whitespace-nowrap">
+                                <div class="flex space-x-1">
+                                    <div class="w-8">{'└ ' + childIdx}</div>
+                                    <ChangeSymbol {changes} item={child} />
+                                </div>
+                            </td>
                             {#each schema as column}
                                 <td class="select-all">{formatValue(child.data[column.name])}</td>
                             {/each}
@@ -122,7 +132,7 @@
     </tbody>
 </table>
 
-{#if onlyRelevantItems && changes.size === 0}
+{#if onlyRelevantItems && changes.changeCount() === 0}
     <div class="flex space-x-1 items-center justify-center text-xl">
         <div>No changes from the transformation.</div>
         <div>Want to see</div>
@@ -144,13 +154,12 @@
 
 <style>
     .page {
-        @apply text-lg;
         @apply font-semibold;
         @apply pr-4;
         @apply whitespace-nowrap;
         position: -webkit-sticky;
         position: sticky;
-        top: 2em;
+        top: 2.4em;
         z-index: 2;
     }
 
@@ -158,7 +167,7 @@
         @apply px-1;
         position: -webkit-sticky;
         position: sticky;
-        top: 2.4em;
+        top: 2.6em;
         z-index: 2;
     }
     th:not(:first-child) {