Fix tests

core/test/Debugger.test.ts

@@ -36,7 +36,7 @@ describe('Transform Items', () => {
     const trans1Items = parsedItems.map((item) => item.withData({ C: `c=${item.value('A')}+${item.value('B')}` }));
 
     const transformers = [new TestTransformer('Trans1', trans1Desc, trans1Schema, trans1Items)];
-    const debug = new Debugger(1, parsedSchema, parsedItems, { fontMap: new Map(), pageViewports: [] }, transformers);
+    const debug = new Debugger(new Map(), [], 1, parsedSchema, parsedItems, transformers);
 
     expect(debug.stageNames).toEqual(['Parse Result', 'Trans1']);
     expect(debug.stageResult(0).schema).toEqual(parsedSchema.map((column) => ({ name: column })));
@@ -62,7 +62,7 @@ describe('Transform Items', () => {
     const trans1Items = parsedItems.map((item) => item.withData({ line: item.data['y'] }));
 
     const transformers = [new TestTransformer('Trans1', trans1Desc, trans1Schema, trans1Items)];
-    const debug = new Debugger(1, parsedSchema, parsedItems, { fontMap: new Map(), pageViewports: [] }, transformers);
+    const debug = new Debugger(new Map(), [], 1, parsedSchema, parsedItems, transformers);
 
     expect(debug.stageNames).toEqual(['Parse Result', 'Trans1']);
     expect(debug.stageResult(0).schema).toEqual([{ name: 'id' }, { name: 'y' }]);
@@ -100,7 +100,7 @@ test('Change inside of Line', async () => {
   const trans1Items = swapElements([...parsedItems], 0, 1);
 
   const transformers = [new TestTransformer('Trans1', trans1Desc, trans1Schema, trans1Items)];
-  const debug = new Debugger(1, parsedSchema, parsedItems, { fontMap: new Map(), pageViewports: [] }, transformers);
+  const debug = new Debugger(new Map(), [], 1, parsedSchema, parsedItems, transformers);
 
   expect(debug.stageNames).toEqual(['Parse Result', 'Trans1']);
   expect(debug.stageResult(0).schema).toEqual([{ name: 'id' }, { name: 'line' }]);
@@ -134,7 +134,7 @@ describe('build schemas', () => {
 
   function calculateSchema(inputSchema: string[], outputSchema: string[]): AnnotatedColumn[] {
     const transformers = [new TestTransformer('Trans1', {}, outputSchema, items)];
-    const debug = new Debugger(1, inputSchema, items, { fontMap: new Map(), pageViewports: [] }, transformers);
+    const debug = new Debugger(new Map(), [], 1, inputSchema, items, transformers);
     return debug.stageResult(1).schema;
   }
 

core/test/Files.test.ts

@@ -83,7 +83,6 @@ function matchFilePath(pdfFileName: string, transformerName: string, chunkCount
 describe('Remove repetitive items from online resources', () => {
   const transformerName = new RemoveRepetitiveItems().name;
   test.each(urls)('URL %p', async (url) => {
-    console.log(url);
     const { fileName, data } = download(url);
     const debug = await pipeline.debug(data, () => {});
     const stageResult = debug.stageResult(debug.stageNames.indexOf(transformerName));
@@ -103,7 +102,6 @@ describe('Remove repetitive items from online resources', () => {
       }),
     );
 
-    console.log(lines);
     const transformerResultAsString = lines.join('\n') || '{}';
     expect(transformerResultAsString).toMatchFile(matchFilePath(fileName, transformerName));
   });
@@ -156,7 +154,6 @@ function itemToString(fontMap: Map<string, object>, item: Item, changeType: stri
 function download(url: string): { fileName: string; data: Buffer } {
   const fileName = path.basename(new URL(url).pathname);
   const localFilePath = `${downloadCache}/${fileName}`;
-  console.log(localFilePath);
   if (!fs.existsSync(localFilePath)) {
     fs.mkdirSync(downloadCache, { recursive: true });
     downloadToFile(url, localFilePath);

core/test/debug/LineItemMerger.test.ts

@@ -1,4 +1,5 @@
 import LineItemMerger from 'src/debug/LineItemMerger';
+import EvaluationTracker from 'src/transformer/EvaluationTracker';
 import ChangeTracker from 'src/debug/ChangeTracker';
 import Item from 'src/Item';
 import { items, realisticItems } from '../testItems';
@@ -6,10 +7,12 @@ import { Addition, ContentChange } from 'src/debug/ChangeIndex';
 
 test('Basics', async () => {
   const itemMerger = new LineItemMerger();
-  const tracker = new ChangeTracker();
+  const evaluationTracker = new EvaluationTracker();
+  const changeTracker = new ChangeTracker();
   expect(itemMerger.groupKey).toEqual('line');
   const mergedItem = itemMerger.merge(
-    tracker,
+    evaluationTracker,
+    changeTracker,
     items(0, [
       {
         line: 2,
@@ -59,19 +62,34 @@ test('Basics', async () => {
 
 test('Track all lines as changes', async () => {
   const itemMerger = new LineItemMerger(true);
-  const tracker = new ChangeTracker();
+  const evaluationTracker = new EvaluationTracker();
-  const mergedItem = itemMerger.merge(tracker, realisticItems(0, [{ line: 1 }, { line: 1 }]));
+  const changeTracker = new ChangeTracker();
-  expect(tracker.change(mergedItem)).toEqual(new Addition());
+  const mergedItem = itemMerger.merge(evaluationTracker, changeTracker, realisticItems(0, [{ line: 1 }, { line: 1 }]));
+  expect(changeTracker.change(mergedItem)).toEqual(new Addition());
+});
+
+test('Mark lines containing evaluated items as evaluated', async () => {
+  const itemMerger = new LineItemMerger();
+  const evaluationTracker = new EvaluationTracker();
+  const changeTracker = new ChangeTracker();
+  const items1 = realisticItems(0, [{ line: 1 }, { line: 1 }]);
+  const items2 = realisticItems(0, [{ line: 2 }, { line: 2 }]);
+  evaluationTracker.trackEvaluation(items1[1]);
+  const mergedItem1 = itemMerger.merge(evaluationTracker, changeTracker, items1);
+  const mergedItem2 = itemMerger.merge(evaluationTracker, changeTracker, items2);
+  expect(evaluationTracker.evaluated(mergedItem1)).toBeTruthy();
+  expect(evaluationTracker.evaluated(mergedItem2)).toBeFalsy();
 });
 
 test('Mark lines containing changed items as changed', async () => {
   const itemMerger = new LineItemMerger();
-  const tracker = new ChangeTracker();
+  const evaluationTracker = new EvaluationTracker();
+  const changeTracker = new ChangeTracker();
   const items1 = realisticItems(0, [{ line: 1 }, { line: 1 }]);
   const items2 = realisticItems(0, [{ line: 2 }, { line: 2 }]);
-  tracker.trackPositionalChange(items1[1], 1, 0);
+  changeTracker.trackPositionalChange(items1[1], 1, 0);
-  const mergedItem1 = itemMerger.merge(tracker, items1);
+  const mergedItem1 = itemMerger.merge(evaluationTracker, changeTracker, items1);
-  const mergedItem2 = itemMerger.merge(tracker, items2);
+  const mergedItem2 = itemMerger.merge(evaluationTracker, changeTracker, items2);
-  expect(tracker.change(mergedItem1)).toEqual(new ContentChange());
+  expect(changeTracker.change(mergedItem1)).toEqual(new ContentChange());
-  expect(tracker.change(mergedItem2)).toEqual(undefined);
+  expect(changeTracker.change(mergedItem2)).toEqual(undefined);
 });

core/test/debug/Page.test.ts

@@ -3,11 +3,13 @@ import Page, { asPages } from 'src/debug/Page';
 import ItemGroup from 'src/debug/ItemGroup';
 import ItemMerger from 'src/debug/ItemMerger';
 import { items } from 'test/testItems';
+import EvaluationTracker from 'src/transformer/EvaluationTracker';
 import ChangeTracker from 'src/debug/ChangeTracker';
 
 test('empty', async () => {
-  const tracker = new ChangeTracker();
+  const evaluationTracker = new EvaluationTracker();
-  expect(asPages(tracker, [])).toEqual([]);
+  const changeTracker = new ChangeTracker();
+  expect(asPages(evaluationTracker, changeTracker, [])).toEqual([]);
 });
 
 test('no merger', async () => {
@@ -21,14 +23,15 @@ test('no merger', async () => {
     items(2, [{ id: 5, line: 1 }]),
   ];
   const flattenedItems = new Array<Item>().concat(...pageItems);
-  const tracker = new ChangeTracker();
+  const evaluationTracker = new EvaluationTracker();
-  const pages = asPages(tracker, flattenedItems);
+  const changeTracker = new ChangeTracker();
+  const pages = asPages(evaluationTracker, changeTracker, flattenedItems);
   expect(pages).toEqual([
     { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) },
     { index: 1, itemGroups: pageItems[1].map((item) => new ItemGroup(item)) },
     { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) },
   ] as Page[]);
-  expect(tracker.changeCount()).toEqual(0);
+  expect(changeTracker.changeCount()).toEqual(0);
 });
 
 test('merger', async () => {
@@ -43,15 +46,19 @@ test('merger', async () => {
   ];
   const flattenedItems = new Array<Item>().concat(...pageItems);
   const merger: ItemMerger = { groupKey: 'line', merge: (items) => items[0] };
-  const tracker = new ChangeTracker();
+  const evaluationTracker = new EvaluationTracker();
-  const pages = asPages(tracker, flattenedItems, merger);
+  const changeTracker = new ChangeTracker();
+  const pages = asPages(evaluationTracker, changeTracker, flattenedItems, merger);
 
   expect(pages).toEqual([
     { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) },
     {
       index: 1,
       itemGroups: [
-        new ItemGroup(merger.merge(tracker, pageItems[1].slice(0, 2)), pageItems[1].slice(0, 2)),
+        new ItemGroup(
+          merger.merge(evaluationTracker, changeTracker, pageItems[1].slice(0, 2)),
+          pageItems[1].slice(0, 2),
+        ),
         new ItemGroup(pageItems[1][2]),
       ],
     },

core/test/debug/StageResults.test.ts

@@ -1,5 +1,6 @@
 import StageResult from 'src/debug/StageResult';
 import { toDescriptor } from 'src/TransformDescriptor';
+import EvaluationTracker from 'src/transformer/EvaluationTracker';
 import ChangeTracker from 'src/debug/ChangeTracker';
 import AnnotatedColumn from 'src/debug/AnnotatedColumn';
 import Page, { asPages } from 'src/debug/Page';
@@ -7,7 +8,8 @@ import { items } from '../testItems';
 import LineItemMerger from 'src/debug/LineItemMerger';
 
 test('itemsUnpacked', async () => {
-  const tracker = new ChangeTracker();
+  const evaluationTracker = new EvaluationTracker();
+  const changeTracker = new ChangeTracker();
   const itemMerger = new LineItemMerger(false);
   const descriptor = toDescriptor({ debug: { itemMerger } });
   const schema: AnnotatedColumn[] = [{ name: 'A' }];
@@ -23,15 +25,16 @@ test('itemsUnpacked', async () => {
       { idx: 5, line: 1 },
     ]),
   ];
-  const pages = asPages(tracker, flatItems, itemMerger);
+  const pages = asPages(evaluationTracker, changeTracker, flatItems, itemMerger);
-  const result = new StageResult(descriptor, schema, pages, tracker, []);
+  const result = new StageResult(descriptor, schema, pages, evaluationTracker, changeTracker, []);
 
   expect(result.itemsUnpacked().map((item) => item.data['idx'])).toEqual([0, 1, 2, 3, 4, 5]);
   expect(result.itemsCleanedAndUnpacked().map((item) => item.data['idx'])).toEqual([0, 1, 2, 3, 4, 5]);
 });
 
 test('itemsCleanedAndUnpacked', async () => {
-  const tracker = new ChangeTracker();
+  const evaluationTracker = new EvaluationTracker();
+  const changeTracker = new ChangeTracker();
   const itemMerger = new LineItemMerger(false);
   const descriptor = toDescriptor({ debug: { itemMerger } });
   const schema: AnnotatedColumn[] = [{ name: 'A' }];
@@ -47,10 +50,10 @@ test('itemsCleanedAndUnpacked', async () => {
       { idx: 5, line: 1 },
     ]),
   ];
-  const pages = asPages(tracker, flatItems, itemMerger);
+  const pages = asPages(evaluationTracker, changeTracker, flatItems, itemMerger);
-  tracker.trackRemoval(flatItems[1]);
+  changeTracker.trackRemoval(flatItems[1]);
-  tracker.trackRemoval(flatItems[4]);
+  changeTracker.trackRemoval(flatItems[4]);
-  const result = new StageResult(descriptor, schema, pages, tracker, []);
+  const result = new StageResult(descriptor, schema, pages, evaluationTracker, changeTracker, []);
 
   expect(result.itemsUnpacked().map((item) => item.data['idx'])).toEqual([0, 1, 2, 3, 4, 5]);
   expect(result.itemsCleanedAndUnpacked().map((item) => item.data['idx'])).toEqual([0, 2, 3, 5]);
@@ -60,8 +63,49 @@ describe('select pages', () => {
   function groupElements(page: Page, elementName: string) {
     return page.itemGroups.map((group) => group.unpacked().map((item) => item.data['idx']));
   }
+
+  test('Evaluation+Changes', async () => {
+    const evaluationTracker = new EvaluationTracker();
+    const changeTracker = new ChangeTracker();
+    const itemMerger = new LineItemMerger(false);
+    const descriptor = toDescriptor({ debug: { itemMerger } });
+    const schema: AnnotatedColumn[] = [{ name: 'A' }];
+    const flatItems = items(0, [
+      { idx: 0, line: 1 }, // nada
+      { idx: 1, line: 2 }, // eval
+      { idx: 2, line: 3 }, // eval + change
+      { idx: 3, line: 4 }, // eval
+      { idx: 4, line: 4 }, // change
+      { idx: 5, line: 4 },
+    ]);
+    evaluationTracker.trackEvaluation(flatItems[1]);
+    evaluationTracker.trackEvaluation(flatItems[2]);
+    evaluationTracker.trackEvaluation(flatItems[3]);
+    changeTracker.trackAddition(flatItems[2]);
+    changeTracker.trackAddition(flatItems[4]);
+    const pages = asPages(evaluationTracker, changeTracker, flatItems, itemMerger);
+    const result = new StageResult(descriptor, schema, pages, evaluationTracker, changeTracker, []);
+
+    const allGrouped = result.selectPages(false, true);
+    expect(allGrouped.map((page) => page.index)).toEqual([0]);
+    expect(groupElements(allGrouped[0], 'idx')).toEqual([[0], [1], [2], [3, 4, 5]]);
+
+    const relevantGrouped = result.selectPages(true, true);
+    expect(relevantGrouped.map((page) => page.index)).toEqual([0]);
+    expect(groupElements(relevantGrouped[0], 'idx')).toEqual([[1], [2], [3, 4, 5]]);
+
+    const relevantUnpacked = result.selectPages(true, false);
+    expect(relevantUnpacked.map((page) => page.index)).toEqual([0]);
+    expect(groupElements(relevantUnpacked[0], 'idx')).toEqual([[1], [2], [3], [4]]);
+
+    const allUnpacked = result.selectPages(false, false);
+    expect(allUnpacked.map((page) => page.index)).toEqual([0]);
+    expect(groupElements(allUnpacked[0], 'idx')).toEqual([[0], [1], [2], [3], [4], [5]]);
+  });
+
   test('Changes on group level', async () => {
-    const tracker = new ChangeTracker();
+    const evaluationTracker = new EvaluationTracker();
+    const changeTracker = new ChangeTracker();
     const itemMerger = new LineItemMerger(true);
     const descriptor = toDescriptor({ debug: { itemMerger } });
     const schema: AnnotatedColumn[] = [{ name: 'A' }];
@@ -77,8 +121,8 @@ describe('select pages', () => {
         { idx: 5, line: 1 },
       ]),
     ];
-    const pages = asPages(tracker, flatItems, itemMerger);
+    const pages = asPages(evaluationTracker, changeTracker, flatItems, itemMerger);
-    const result = new StageResult(descriptor, schema, pages, tracker, []);
+    const result = new StageResult(descriptor, schema, pages, evaluationTracker, changeTracker, []);
 
     const allGrouped = result.selectPages(false, true);
     expect(allGrouped.map((page) => page.index)).toEqual([0, 1, 2]);
@@ -106,7 +150,8 @@ describe('select pages', () => {
   });
 
   test('Changes on element level', async () => {
-    const tracker = new ChangeTracker();
+    const evaluationTracker = new EvaluationTracker();
+    const changeTracker = new ChangeTracker();
     const itemMerger = new LineItemMerger(false);
     const descriptor = toDescriptor({ debug: { itemMerger } });
     const schema: AnnotatedColumn[] = [{ name: 'A' }];
@@ -123,10 +168,10 @@ describe('select pages', () => {
         { idx: 6, line: 2 },
       ]),
     ];
-    tracker.trackAddition(flatItems[3]);
+    changeTracker.trackAddition(flatItems[3]);
-    tracker.trackAddition(flatItems[5]);
+    changeTracker.trackAddition(flatItems[5]);
-    const pages = asPages(tracker, flatItems, itemMerger);
+    const pages = asPages(evaluationTracker, changeTracker, flatItems, itemMerger);
-    const result = new StageResult(descriptor, schema, pages, tracker, []);
+    const result = new StageResult(descriptor, schema, pages, evaluationTracker, changeTracker, []);
 
     const allGrouped = result.selectPages(false, true);
     expect(allGrouped.map((page) => page.index)).toEqual([0, 1, 2]);
@@ -153,8 +198,9 @@ describe('select pages', () => {
     expect(groupElements(allUnpacked[2], 'idx')).toEqual([[4], [5], [6]]);
   });
 
-  test('showAll - grouped', async () => {
+  test('showAll - grouped - merger', async () => {
-    const tracker = new ChangeTracker();
+    const evaluationTracker = new EvaluationTracker();
+    const changeTracker = new ChangeTracker();
     const itemMerger = new LineItemMerger(false);
     const descriptor = toDescriptor({ debug: { itemMerger, showAll: true } });
     const schema: AnnotatedColumn[] = [{ name: 'A' }];
@@ -170,8 +216,8 @@ describe('select pages', () => {
         { idx: 5, line: 1 },
       ]),
     ];
-    const pages = asPages(tracker, flatItems, itemMerger);
+    const pages = asPages(evaluationTracker, changeTracker, flatItems, itemMerger);
-    const result = new StageResult(descriptor, schema, pages, tracker, []);
+    const result = new StageResult(descriptor, schema, pages, evaluationTracker, changeTracker, []);
 
     const relevantGrouped = result.selectPages(true, true);
     expect(relevantGrouped.map((page) => page.index)).toEqual([0, 1, 2]);
@@ -180,8 +226,9 @@ describe('select pages', () => {
     expect(groupElements(relevantGrouped[2], 'idx')).toEqual([[4, 5]]);
   });
 
-  test('showAll - grouped', async () => {
+  test('showAll - grouped - no merger', async () => {
-    const tracker = new ChangeTracker();
+    const evaluationTracker = new EvaluationTracker();
+    const changeTracker = new ChangeTracker();
     const descriptor = toDescriptor({ debug: { showAll: true } });
     const schema: AnnotatedColumn[] = [{ name: 'A' }];
     const flatItems = [
@@ -189,8 +236,8 @@ describe('select pages', () => {
       ...items(1, [{ idx: 3 }]),
       ...items(2, [{ idx: 4 }, { idx: 5 }]),
     ];
-    const pages = asPages(tracker, flatItems);
+    const pages = asPages(evaluationTracker, changeTracker, flatItems);
-    const result = new StageResult(descriptor, schema, pages, tracker, []);
+    const result = new StageResult(descriptor, schema, pages, evaluationTracker, changeTracker, []);
 
     const relevantGrouped = result.selectPages(true, true);
     expect(relevantGrouped.map((page) => page.index)).toEqual([0, 1, 2]);

core/test/transformer/testContext.ts

@@ -1,8 +1,5 @@
-import type TransformContext from "src/transformer/TransformContext";
+import TransformContext from 'src/transformer/TransformContext';
 
-export function emptyContext():TransformContext{
+export function emptyContext(): TransformContext {
-    return {
+  return new TransformContext(new Map(), []);
-        fontMap:new Map(),
-        pageViewports:[]
-    };
 }