From e7574513c56975cb26c97b86eabe5cd00ee47574 Mon Sep 17 00:00:00 2001 From: Johannes Zillmann Date: Sun, 28 Feb 2021 02:07:45 +0100 Subject: [PATCH] Change detection on group and item level --- core/src/Debugger.ts | 48 ++++--- core/src/TransformDescriptor.ts | 2 +- core/src/assert.ts | 6 + core/src/debug/ChangeIndex.ts | 78 ++++++++++++ core/src/debug/ChangeTracker.ts | 71 +++++++++++ core/src/{support => debug}/ItemGroup.ts | 9 +- core/src/{support => debug}/ItemMerger.ts | 3 +- core/src/{support => debug}/LineItemMerger.ts | 14 ++- core/src/debug/Page.ts | 29 +++++ core/src/debug/StageResult.ts | 41 ++++-- core/src/debug/detectChanges.ts | 55 ++------ core/src/support/Page.ts | 6 - .../{itemUtils.ts => groupingUtils.ts} | 26 +--- core/src/transformer/AdjustHeight.ts | 3 +- core/src/transformer/CompactLines.ts | 6 +- core/src/transformer/SortXWithinLines.ts | 4 +- core/test/Debugger.test.ts | 117 +++++++++++++++--- core/test/debug/LineItemMerger.test.ts | 33 ++++- core/test/debug/Page.test.ts | 60 +++++++++ core/test/debug/detectChanges.test.ts | 21 ++-- ...temUtils.test.ts => groupingUtils.test.ts} | 49 +------- core/test/testItems.ts | 17 +++ ui/src/components/ComponentDefinition.ts | 5 + ui/src/debug/ChangeSymbol.svelte | 49 +++++--- ui/src/debug/DebugView.svelte | 6 +- ui/src/debug/ItemTable.svelte | 45 ++++--- 26 files changed, 562 insertions(+), 241 deletions(-) create mode 100644 core/src/debug/ChangeIndex.ts create mode 100644 core/src/debug/ChangeTracker.ts rename core/src/{support => debug}/ItemGroup.ts (60%) rename core/src/{support => debug}/ItemMerger.ts (64%) rename core/src/{support => debug}/LineItemMerger.ts (69%) create mode 100644 core/src/debug/Page.ts delete mode 100644 core/src/support/Page.ts rename core/src/support/{itemUtils.ts => groupingUtils.ts} (66%) create mode 100644 core/test/debug/Page.test.ts rename core/test/support/{itemUtils.test.ts => groupingUtils.test.ts} (62%) create mode 100644 ui/src/components/ComponentDefinition.ts diff --git a/core/src/Debugger.ts b/core/src/Debugger.ts index 6593c63..5504529 100644 --- a/core/src/Debugger.ts +++ b/core/src/Debugger.ts @@ -1,10 +1,12 @@ import Item from './Item'; import ItemTransformer from './transformer/ItemTransformer'; import TransformContext from './transformer/TransformContext'; -import StageResult from './debug/StageResult'; +import StageResult, { initialStage } from './debug/StageResult'; import ColumnAnnotation from './debug/ColumnAnnotation'; import AnnotatedColumn from './debug/AnnotatedColumn'; import { detectChanges } from './debug/detectChanges'; +import { asPages } from './debug/Page'; +import ChangeTracker from './debug/ChangeTracker'; export default class Debugger { private context: TransformContext; @@ -16,42 +18,36 @@ export default class Debugger { this.transformers = transformers; this.context = context; this.stageNames = ['Parse Result', ...transformers.map((t) => t.name)]; - this.stageResultCache = [ - { - schema: inputSchema.map((column) => ({ name: column })), - items: inputItems, - changes: new Map(), - messages: [ - `Parsed ${inputItems.length === 0 ? 0 : inputItems[inputItems.length - 1].page + 1} pages with ${ - inputItems.length - } items`, - ], - }, - ]; + this.stageResultCache = [initialStage(inputSchema, inputItems)]; } - //TODO return MarkedItem ? (removed, added, etc..)? stageResults(stageIndex: number): StageResult { for (let idx = 0; idx < stageIndex + 1; idx++) { if (!this.stageResultCache[idx]) { const transformer = this.transformers[idx - 1]; const previousStageResult: StageResult = this.stageResultCache[idx - 1]; + const previousItems = previousStageResult.itemsUnpacked(); const inputSchema = toSimpleSchema(previousStageResult); const outputSchema = transformer.schemaTransformer(inputSchema); - const itemResult = transformer.transform(this.context, [...previousStageResult.items]); + const itemResult = transformer.transform(this.context, [...previousItems]); - const changes = detectChanges(previousStageResult.items, itemResult.items); - - const stageResult = { - descriptor: transformer.descriptor, - schema: toAnnotatedSchema(inputSchema, outputSchema), - ...itemResult, - changes, - }; - if (changes.size > 0) { - stageResult.messages.unshift(`Detected ${changes.size} changes`); + const changeTracker = new ChangeTracker(); + detectChanges(changeTracker, previousItems, itemResult.items); + const pages = asPages(changeTracker, itemResult.items, transformer.descriptor.debug?.itemMerger); + const messages = itemResult.messages; + if (changeTracker.changeCount() > 0) { + messages.unshift(`Detected ${changeTracker.changeCount()} changes`); } - this.stageResultCache.push(stageResult); + + this.stageResultCache.push( + new StageResult( + transformer.descriptor, + toAnnotatedSchema(inputSchema, outputSchema), + pages, + changeTracker, + messages, + ), + ); } } return this.stageResultCache[stageIndex]; diff --git a/core/src/TransformDescriptor.ts b/core/src/TransformDescriptor.ts index 616540f..58f715a 100644 --- a/core/src/TransformDescriptor.ts +++ b/core/src/TransformDescriptor.ts @@ -1,4 +1,4 @@ -import type ItemMerger from './support/ItemMerger'; +import type ItemMerger from './debug/ItemMerger'; interface Debug { /** diff --git a/core/src/assert.ts b/core/src/assert.ts index 207c706..dadce11 100644 --- a/core/src/assert.ts +++ b/core/src/assert.ts @@ -4,6 +4,12 @@ export function assert(condition: boolean, message: string) { } } +export function assertNot(condition: boolean, message: string) { + if (condition) { + throw new Error(message || 'Assertion failed'); + } +} + export function assertDefined(value: T | undefined, message: string): T { if (value === null || typeof value === 'undefined') { throw new Error(message || 'Assertion failed'); diff --git a/core/src/debug/ChangeIndex.ts b/core/src/debug/ChangeIndex.ts new file mode 100644 index 0000000..bee1f2c --- /dev/null +++ b/core/src/debug/ChangeIndex.ts @@ -0,0 +1,78 @@ +import Item from '../Item'; + +export default interface ChangeIndex { + /** + * Return the number of tracked changes. + */ + changeCount(): number; + + /** + * Returns the change if for the given item if there is any + * @param item + */ + change(item: Item): Change | undefined; + + /** + * Returs true if the given item has changed + * @param item + */ + hasChanged(item: Item): boolean; + + /** + * Returns true if there is a change and it's category is ChangeCategory.PLUS + * @param item + */ + isPlusChange(item: Item): boolean; + + /** + * Returns true if there is a change and it's category is ChangeCategory.NEUTRAL + * @param item + */ + isNeutralChange(item: Item): boolean; + + /** + * Returns true if there is a change and it's category is ChangeCategory.MINUS + * @param item + */ + isMinusChange(item: Item): boolean; +} + +export abstract class Change { + constructor(public category: ChangeCategory) {} +} + +// This is merely for coloring different kind of changes +export enum ChangeCategory { + PLUS = 'PLUS', + MINUS = 'MINUS', + NEUTRAL = 'NEUTRAL', +} + +export class Addition extends Change { + constructor() { + super(ChangeCategory.PLUS); + } +} + +export class Removal extends Change { + constructor() { + super(ChangeCategory.MINUS); + } +} + +export class ContentChange extends Change { + constructor() { + super(ChangeCategory.NEUTRAL); + } +} + +export enum Direction { + UP = 'UP', + DOWN = 'DOWN', +} + +export class PositionChange extends Change { + constructor(public direction: Direction, public amount: number) { + super(direction === Direction.UP ? ChangeCategory.PLUS : ChangeCategory.MINUS); + } +} diff --git a/core/src/debug/ChangeTracker.ts b/core/src/debug/ChangeTracker.ts new file mode 100644 index 0000000..74761ee --- /dev/null +++ b/core/src/debug/ChangeTracker.ts @@ -0,0 +1,71 @@ +import ChangeIndex, { + Change, + ChangeCategory, + Addition, + Removal, + PositionChange, + ContentChange, + Direction, +} from './ChangeIndex'; +import Item from '../Item'; +import { assertNot, assertDefined } from '../assert'; + +const ADDITION = new Addition(); +const REMOVAL = new Removal(); +const CONTENT_CHANGE = new ContentChange(); + +export default class ChangeTracker implements ChangeIndex { + private changes: Map = new Map(); + + private addChange(item: Item, change: Change) { + const uuid = _uuid(item); + assertNot(this.changes.has(uuid), `Change for item ${uuid} already defined`); + this.changes.set(uuid, change); + } + + changeCount(): number { + return this.changes.size; + } + + trackAddition(item: Item) { + this.addChange(item, ADDITION); + } + + trackRemoval(item: Item) { + this.addChange(item, REMOVAL); + } + + trackPositionalChange(item: Item, oldPosition: number, newPosition: number) { + const direction = newPosition > oldPosition ? Direction.DOWN : Direction.UP; + const amount = Math.abs(newPosition - oldPosition); + this.addChange(item, new PositionChange(direction, amount)); + } + + trackContentChange(item: Item) { + this.addChange(item, CONTENT_CHANGE); + } + + change(item: Item): Change | undefined { + return this.changes.get(_uuid(item)); + } + + hasChanged(item: Item): boolean { + return this.changes.has(_uuid(item)); + } + + isPlusChange(item: Item): boolean { + return this.change(item)?.category === ChangeCategory.PLUS; + } + + isNeutralChange(item: Item): boolean { + return this.change(item)?.category === ChangeCategory.NEUTRAL; + } + + isMinusChange(item: Item): boolean { + return this.change(item)?.category === ChangeCategory.MINUS; + } +} + +function _uuid(item: Item): string { + return assertDefined(item.uuid, 'UUID is not set'); +} diff --git a/core/src/support/ItemGroup.ts b/core/src/debug/ItemGroup.ts similarity index 60% rename from core/src/support/ItemGroup.ts rename to core/src/debug/ItemGroup.ts index 5dbc7ed..04a8b1e 100644 --- a/core/src/support/ItemGroup.ts +++ b/core/src/debug/ItemGroup.ts @@ -1,4 +1,4 @@ -import type Item from '../Item'; +import type Item from '..//Item'; export default class ItemGroup { top: Item; @@ -12,4 +12,11 @@ export default class ItemGroup { hasMany(): boolean { return this.elements.length > 0; } + + unpacked(): Item[] { + if (this.elements.length > 0) { + return this.elements; + } + return [this.top]; + } } diff --git a/core/src/support/ItemMerger.ts b/core/src/debug/ItemMerger.ts similarity index 64% rename from core/src/support/ItemMerger.ts rename to core/src/debug/ItemMerger.ts index f3901e6..d18b9bb 100644 --- a/core/src/support/ItemMerger.ts +++ b/core/src/debug/ItemMerger.ts @@ -1,3 +1,4 @@ +import type ChangeTracker from './ChangeTracker'; import type Item from '../Item'; /** @@ -5,5 +6,5 @@ import type Item from '../Item'; */ export default abstract class ItemMerger { constructor(public groupKey: string) {} - abstract merge(items: Item[]): Item; + abstract merge(tracker: ChangeTracker, items: Item[]): Item; } diff --git a/core/src/support/LineItemMerger.ts b/core/src/debug/LineItemMerger.ts similarity index 69% rename from core/src/support/LineItemMerger.ts rename to core/src/debug/LineItemMerger.ts index 2f5ae61..cd80117 100644 --- a/core/src/support/LineItemMerger.ts +++ b/core/src/debug/LineItemMerger.ts @@ -1,12 +1,13 @@ import ItemMerger from './ItemMerger'; import Item from '../Item'; +import ChangeTracker from './ChangeTracker'; export default class LineItemMerger extends ItemMerger { - constructor() { + constructor(private trackAsNew = false) { super('line'); } - merge(items: Item[]): Item { + merge(tracker: ChangeTracker, items: Item[]): Item { const page = items[0].page; const line = items[0].data['line']; const str = items.map((item) => item.data['str']).join(' '); @@ -16,7 +17,7 @@ export default class LineItemMerger extends ItemMerger { const height = Math.max(...items.map((item) => item.data['height'])); const fontNames = [...new Set(items.map((item) => item.data['fontName']))]; const directions = [...new Set(items.map((item) => item.data['dir']))]; - return new Item(page, { + const newItem = new Item(page, { str, line, x, @@ -26,5 +27,12 @@ export default class LineItemMerger extends ItemMerger { fontName: fontNames, dir: directions, }); + + if (this.trackAsNew) { + tracker.trackAddition(newItem); + } else if (items.find((item) => tracker.hasChanged(item))) { + tracker.trackContentChange(newItem); + } + return newItem; } } diff --git a/core/src/debug/Page.ts b/core/src/debug/Page.ts new file mode 100644 index 0000000..e4aabf8 --- /dev/null +++ b/core/src/debug/Page.ts @@ -0,0 +1,29 @@ +import Item from '../Item'; +import { groupByElement, groupByPage } from '../support/groupingUtils'; +import ChangeTracker from './ChangeTracker'; +import ItemGroup from './ItemGroup'; +import ItemMerger from './ItemMerger'; + +export default interface Page { + index: number; + itemGroups: ItemGroup[]; +} + +export function asPages(tracker: ChangeTracker, items: Item[], itemMerger?: ItemMerger): Page[] { + return groupByPage(items).map((pageItems: Item[]) => { + let itemGroups: ItemGroup[]; + if (itemMerger) { + itemGroups = groupByElement(pageItems, itemMerger.groupKey).map((groupItems) => { + if (groupItems.length > 1) { + const top = itemMerger.merge(tracker, groupItems); + return new ItemGroup(top, groupItems); + } else { + return new ItemGroup(groupItems[0]); + } + }); + } else { + itemGroups = pageItems.map((item) => new ItemGroup(item)); + } + return { index: pageItems[0].page, itemGroups } as Page; + }); +} diff --git a/core/src/debug/StageResult.ts b/core/src/debug/StageResult.ts index ea406c4..17cadce 100644 --- a/core/src/debug/StageResult.ts +++ b/core/src/debug/StageResult.ts @@ -1,12 +1,35 @@ -import TransformDescriptor from '../TransformDescriptor'; -import Item from '../Item'; +import TransformDescriptor, { toDescriptor } from '../TransformDescriptor'; import AnnotatedColumn from './AnnotatedColumn'; -import { Change } from './detectChanges'; +import Item from '../Item'; +import Page, { asPages } from './Page'; +import ChangeIndex from './ChangeIndex'; +import ChangeTracker from './ChangeTracker'; -export default interface StageResult { - descriptor?: TransformDescriptor; - schema: AnnotatedColumn[]; - items: Item[]; - changes: Map; - messages: string[]; +export default class StageResult { + constructor( + public descriptor: TransformDescriptor, + public schema: AnnotatedColumn[], + public pages: Page[], + public changes: ChangeIndex, + public messages: string[], + ) {} + + itemsUnpacked(): Item[] { + return this.pages.reduce((items: Item[], page: Page) => { + page.itemGroups.forEach((itemGroup) => itemGroup.unpacked().forEach((item) => items.push(item))); + return items; + }, []); + } +} + +export function initialStage(inputSchema: string[], inputItems: Item[]): StageResult { + const schema = inputSchema.map((column) => ({ name: column })); + const tracker = new ChangeTracker(); + const pages = asPages(tracker, inputItems); + const messages = [ + `Parsed ${inputItems.length === 0 ? 0 : inputItems[inputItems.length - 1].page + 1} pages with ${ + inputItems.length + } items`, + ]; + return new StageResult(toDescriptor({}), schema, pages, tracker, messages); } diff --git a/core/src/debug/detectChanges.ts b/core/src/debug/detectChanges.ts index 15c7ee4..001d211 100644 --- a/core/src/debug/detectChanges.ts +++ b/core/src/debug/detectChanges.ts @@ -1,67 +1,26 @@ +import ChangeTracker from './ChangeTracker'; import { assertDefined } from '../assert'; import Item from '../Item'; -// export enum ChangeType { -// POSITION_CHANGED = 'POSITION_CHANGED', -// ADDED = 'ADDED', -// REMOVED = 'REMOVED', -// CHANGED = 'CHANGED', -// EVALUATED = 'EVALUATED', -// } - -// Marker interface for a change -export abstract class Change { - constructor(public category: ChangeCategory) {} -} - -// This is merely for coloring different kind of changes -export enum ChangeCategory { - PLUS = 'PLUS', - MINUS = 'MINUS', - NEUTRAL = 'NEUTRAL', -} - -export class Addition extends Change { - constructor() { - super(ChangeCategory.PLUS); - } -} - -export enum Direction { - UP = 'UP', - DOWN = 'DOWN', -} - -export class PositionChange extends Change { - constructor(public direction: Direction, public amount: number) { - super(direction === Direction.UP ? ChangeCategory.PLUS : ChangeCategory.NEUTRAL); - } -} - /** * Compares incomming and outgoing items of a transformer in order to detect changes and to display them in any debug visualization. */ -export function detectChanges(inputItems: Item[], transformedItems: Item[]): Map { - const changes: Map = new Map(); +export function detectChanges(tracker: ChangeTracker, inputItems: Item[], transformedItems: Item[]) { const inputItemsByUuid = inputMap(inputItems); transformedItems.forEach((item, idx) => { - const uuid = getUuid(item); + const uuid = _uuid(item); const oldItem = inputItemsByUuid.get(uuid); if (oldItem) { if (idx !== oldItem.position) { - const direction = idx > oldItem.position ? Direction.DOWN : Direction.UP; - const amount = Math.abs(idx - oldItem.position); - changes.set(uuid, new PositionChange(direction, amount)); + tracker.trackPositionalChange(item, oldItem.position, idx); } //TODO check for change } else { - changes.set(uuid, new Addition()); + tracker.trackAddition(item); } }); //TODO detect removals (need to re-add them ?) - - return changes; } interface InputItem { @@ -71,11 +30,11 @@ interface InputItem { function inputMap(inputItems: Item[]): Map { return inputItems.reduce((map, item, idx) => { - map.set(getUuid(item), { item, position: idx }); + map.set(_uuid(item), { item, position: idx }); return map; }, new Map()); } -function getUuid(item: Item): string { +function _uuid(item: Item): string { return assertDefined(item.uuid, 'UUID is not set'); } diff --git a/core/src/support/Page.ts b/core/src/support/Page.ts deleted file mode 100644 index ca04b7a..0000000 --- a/core/src/support/Page.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type ItemGroup from './ItemGroup'; - -export default interface Page { - index: number; - itemGroups: ItemGroup[]; -} diff --git a/core/src/support/itemUtils.ts b/core/src/support/groupingUtils.ts similarity index 66% rename from core/src/support/itemUtils.ts rename to core/src/support/groupingUtils.ts index 92503bd..553cb9f 100644 --- a/core/src/support/itemUtils.ts +++ b/core/src/support/groupingUtils.ts @@ -1,7 +1,8 @@ -import ItemMerger from './ItemMerger'; +import ItemMerger from '../debug/ItemMerger'; import Item from '../Item'; -import ItemGroup from './ItemGroup'; -import Page from './Page'; +import ItemGroup from '../debug/ItemGroup'; +import Page from '../debug/Page'; +import ChangeTracker from '../debug/ChangeTracker'; type KeyExtractor = (item: Item) => any; type PageItemTransformer = (page: number, items: Item[]) => Item[]; @@ -42,22 +43,3 @@ export function transformGroupedByPageAndLine(items: Item[], groupedTransformer: }); return transformedItems; } - -export function asPages(items: Item[], itemMerger?: ItemMerger): Page[] { - return groupByPage(items).map((pageItems: Item[]) => { - let itemGroups: ItemGroup[]; - if (itemMerger) { - itemGroups = groupByElement(pageItems, itemMerger.groupKey).map((groupItems) => { - if (groupItems.length > 1) { - const top = itemMerger.merge(groupItems); - return new ItemGroup(top, groupItems); - } else { - return new ItemGroup(groupItems[0]); - } - }); - } else { - itemGroups = pageItems.map((item) => new ItemGroup(item)); - } - return { index: pageItems[0].page, itemGroups } as Page; - }); -} diff --git a/core/src/transformer/AdjustHeight.ts b/core/src/transformer/AdjustHeight.ts index e2cc78c..b9c9324 100644 --- a/core/src/transformer/AdjustHeight.ts +++ b/core/src/transformer/AdjustHeight.ts @@ -1,9 +1,8 @@ -import PageViewport from '../parse/PageViewport'; import Item from '../Item'; import ItemResult from '../ItemResult'; import ItemTransformer from './ItemTransformer'; import TransformContext from './TransformContext'; -import { transformGroupedByPage } from '../support/itemUtils'; +import { transformGroupedByPage } from '../support/groupingUtils'; export default class AdjustHeight extends ItemTransformer { constructor() { diff --git a/core/src/transformer/CompactLines.ts b/core/src/transformer/CompactLines.ts index a184b3d..fc1c1bf 100644 --- a/core/src/transformer/CompactLines.ts +++ b/core/src/transformer/CompactLines.ts @@ -2,8 +2,8 @@ import Item from '../Item'; import ItemResult from '../ItemResult'; import ItemTransformer from './ItemTransformer'; import TransformContext from './TransformContext'; -import { transformGroupedByPage } from '../support/itemUtils'; -import LineItemMerger from '../support/LineItemMerger'; +import LineItemMerger from '../debug/LineItemMerger'; +import { transformGroupedByPage } from '../support/groupingUtils'; export default class CompactLines extends ItemTransformer { constructor() { @@ -13,7 +13,7 @@ export default class CompactLines extends ItemTransformer { { requireColumns: ['str', 'y', 'height'], debug: { - itemMerger: new LineItemMerger(), + itemMerger: new LineItemMerger(true), }, }, (incomingSchema) => { diff --git a/core/src/transformer/SortXWithinLines.ts b/core/src/transformer/SortXWithinLines.ts index fdd1ec8..3813ca6 100644 --- a/core/src/transformer/SortXWithinLines.ts +++ b/core/src/transformer/SortXWithinLines.ts @@ -2,8 +2,8 @@ import Item from '../Item'; import ItemResult from '../ItemResult'; import ItemTransformer from './ItemTransformer'; import TransformContext from './TransformContext'; -import LineItemMerger from '../support/LineItemMerger'; -import { transformGroupedByPageAndLine } from '../support/itemUtils'; +import LineItemMerger from '../debug/LineItemMerger'; +import { transformGroupedByPageAndLine } from '../support/groupingUtils'; export default class SortXWithinLines extends ItemTransformer { constructor() { diff --git a/core/test/Debugger.test.ts b/core/test/Debugger.test.ts index 850dc57..a9aa9fc 100644 --- a/core/test/Debugger.test.ts +++ b/core/test/Debugger.test.ts @@ -3,17 +3,19 @@ import Item from 'src/Item'; import ItemTransformer from 'src/transformer/ItemTransformer'; import TransformDescriptor from 'src/TransformDescriptor'; import TransformContext from 'src/transformer/TransformContext'; +import LineItemMerger from 'src/debug/LineItemMerger'; import ItemResult from 'src/ItemResult'; import ColumnAnnotation from 'src/debug/ColumnAnnotation'; import AnnotatedColumn from 'src/debug/AnnotatedColumn'; +import { items } from './testItems'; class TestTransformer extends ItemTransformer { items: Item[]; constructor(name: string, descriptor: Partial, outputSchema: string[], items: Item[]) { - super(name, `Description for ${name}`, descriptor, (incomingSchema) => outputSchema); + super(name, `Description for ${name}`, descriptor, (_) => outputSchema); this.items = items; } - transform(_: TransformContext, items: Item[]): ItemResult { + transform(_: TransformContext, __: Item[]): ItemResult { return { items: this.items, messages: [], @@ -21,27 +23,112 @@ class TestTransformer extends ItemTransformer { } } -test('basic debug', async () => { - const parsedSchema = ['A', 'B']; - const parsedItems = [new Item(0, { A: 'a_row1', B: 'b_row1' }), new Item(0, { A: 'a_row2', B: 'b_row2' })]; +describe('Transform Items', () => { + test('Basics', async () => { + const parsedSchema = ['A', 'B']; + const parsedItems = items(0, [ + { A: 'a_row1', B: 'b_row1' }, + { A: 'a_row2', B: 'b_row2' }, + ]); - const trans1Desc = { requireColumns: ['A', 'B'] }; - const trans1Schema = ['C']; - const trans1Items = parsedItems.map((item) => item.withData({ C: `c=${item.value('A')}+${item.value('B')}` })); + const trans1Desc = { requireColumns: ['A', 'B'] }; + const trans1Schema = ['C']; + const trans1Items = parsedItems.map((item) => item.withData({ C: `c=${item.value('A')}+${item.value('B')}` })); + + const transformers = [new TestTransformer('Trans1', trans1Desc, trans1Schema, trans1Items)]; + const debug = new Debugger(parsedSchema, parsedItems, { fontMap: new Map(), pageViewports: [] }, transformers); + + expect(debug.stageNames).toEqual(['Parse Result', 'Trans1']); + expect(debug.stageResults(0).schema).toEqual(parsedSchema.map((column) => ({ name: column }))); + expect(debug.stageResults(1).schema).toEqual([ + ...parsedSchema.map((column) => ({ name: column, annotation: ColumnAnnotation.REMOVED })), + { name: 'C', annotation: ColumnAnnotation.ADDED }, + ]); + + expect(debug.stageResults(0).itemsUnpacked()).toEqual(parsedItems); + expect(debug.stageResults(1).itemsUnpacked()).toEqual(trans1Items); + }); + + test('Line Merge', async () => { + const parsedSchema = ['id', 'y']; + const parsedItems = items(0, [ + { id: 1, y: 1 }, + { id: 2, y: 2 }, + { id: 3, y: 2 }, + ]); + + const trans1Desc = { requireColumns: ['id', 'y'], debug: { itemMerger: new LineItemMerger(true) } }; + const trans1Schema = ['id', 'line']; + const trans1Items = parsedItems.map((item) => item.withData({ line: item.data['y'] })); + + const transformers = [new TestTransformer('Trans1', trans1Desc, trans1Schema, trans1Items)]; + const debug = new Debugger(parsedSchema, parsedItems, { fontMap: new Map(), pageViewports: [] }, transformers); + + expect(debug.stageNames).toEqual(['Parse Result', 'Trans1']); + expect(debug.stageResults(0).schema).toEqual([{ name: 'id' }, { name: 'y' }]); + expect(debug.stageResults(1).schema).toEqual([ + { name: 'id' }, + { name: 'y', annotation: ColumnAnnotation.REMOVED }, + { name: 'line', annotation: ColumnAnnotation.ADDED }, + ]); + + expect(debug.stageResults(0).itemsUnpacked()).toEqual(parsedItems); + expect(debug.stageResults(1).itemsUnpacked()).toEqual(trans1Items); + + const lineMergingStage = debug.stageResults(1); + const { changes, pages } = lineMergingStage; + + //verify item groups + expect(pages[0].itemGroups.map((itemGroup) => changes.hasChanged(itemGroup.top))).toEqual([false, true]); + + //verify unpacked items + expect(lineMergingStage.itemsUnpacked().map((item) => changes.hasChanged(item))).toEqual([false, false, false]); + }); +}); + +test('Change inside of Line', async () => { + const parsedSchema = ['id', 'line']; + const parsedItems = items(0, [ + { id: 1, line: 1 }, + { id: 2, line: 1 }, + { id: 3, line: 2 }, + { id: 4, line: 2 }, + ]); + + const trans1Desc = { requireColumns: ['id', 'line'], debug: { itemMerger: new LineItemMerger() } }; + const trans1Schema = parsedSchema; + const trans1Items = swapElements([...parsedItems], 0, 1); const transformers = [new TestTransformer('Trans1', trans1Desc, trans1Schema, trans1Items)]; const debug = new Debugger(parsedSchema, parsedItems, { fontMap: new Map(), pageViewports: [] }, transformers); expect(debug.stageNames).toEqual(['Parse Result', 'Trans1']); - expect(debug.stageResults(0).schema).toEqual(parsedSchema.map((column) => ({ name: column }))); - expect(debug.stageResults(1).schema).toEqual([ - ...parsedSchema.map((column) => ({ name: column, annotation: ColumnAnnotation.REMOVED })), - { name: 'C', annotation: ColumnAnnotation.ADDED }, - ]); - expect(debug.stageResults(0).items).toEqual(parsedItems); - expect(debug.stageResults(1).items).toEqual(trans1Items); + expect(debug.stageResults(0).schema).toEqual([{ name: 'id' }, { name: 'line' }]); + expect(debug.stageResults(1).schema).toEqual([{ name: 'id' }, { name: 'line' }]); + expect(debug.stageResults(0).itemsUnpacked()).toEqual(parsedItems); + expect(debug.stageResults(1).itemsUnpacked()).toEqual(trans1Items); + + const { changes, pages } = debug.stageResults(1); + + //verify item groups + expect(pages[0].itemGroups.map((itemGroup) => changes.hasChanged(itemGroup.top))).toEqual([true, false]); + + //verify unpacked items + expect( + debug + .stageResults(1) + .itemsUnpacked() + .map((item) => changes.hasChanged(item)), + ).toEqual([true, true, false, false]); }); +var swapElements = function (arr: Item[], indexA: number, indexB: number): Item[] { + var temp = arr[indexA]; + arr[indexA] = arr[indexB]; + arr[indexB] = temp; + return arr; +}; + describe('build schemas', () => { const items: Item[] = []; diff --git a/core/test/debug/LineItemMerger.test.ts b/core/test/debug/LineItemMerger.test.ts index a5cdf02..47ac2c3 100644 --- a/core/test/debug/LineItemMerger.test.ts +++ b/core/test/debug/LineItemMerger.test.ts @@ -1,13 +1,15 @@ import LineItemMerger from 'src/debug/LineItemMerger'; +import ChangeTracker from 'src/debug/ChangeTracker'; import Item from 'src/Item'; -import { items } from '../testItems'; - -const itemMerger = new LineItemMerger(); +import { items, realisticItems } from '../testItems'; +import { Addition, ContentChange } from 'src/debug/ChangeIndex'; test('Basics', async () => { + const itemMerger = new LineItemMerger(); + const tracker = new ChangeTracker(); expect(itemMerger.groupKey).toEqual('line'); - - const mergedItem = itemMerger?.merge( + const mergedItem = itemMerger.merge( + tracker, items(0, [ { line: 2, @@ -41,7 +43,7 @@ test('Basics', async () => { }, ]), ); - expect(mergedItem?.withoutUuid()).toEqual( + expect(mergedItem.withoutUuid()).toEqual( new Item(0, { line: 2, x: 240, @@ -54,3 +56,22 @@ test('Basics', async () => { }).withoutUuid(), ); }); + +test('Track all lines as changes', async () => { + const itemMerger = new LineItemMerger(true); + const tracker = new ChangeTracker(); + const mergedItem = itemMerger.merge(tracker, realisticItems(0, [{ line: 1 }, { line: 1 }])); + expect(tracker.change(mergedItem)).toEqual(new Addition()); +}); + +test('Mark lines containing changed items as changed', async () => { + const itemMerger = new LineItemMerger(); + const tracker = new ChangeTracker(); + const items1 = realisticItems(0, [{ line: 1 }, { line: 1 }]); + const items2 = realisticItems(0, [{ line: 2 }, { line: 2 }]); + tracker.trackPositionalChange(items1[1], 1, 0); + const mergedItem1 = itemMerger.merge(tracker, items1); + const mergedItem2 = itemMerger.merge(tracker, items2); + expect(tracker.change(mergedItem1)).toEqual(new ContentChange()); + expect(tracker.change(mergedItem2)).toEqual(undefined); +}); diff --git a/core/test/debug/Page.test.ts b/core/test/debug/Page.test.ts new file mode 100644 index 0000000..0c7fe90 --- /dev/null +++ b/core/test/debug/Page.test.ts @@ -0,0 +1,60 @@ +import Item from 'src/Item'; +import Page, { asPages } from 'src/debug/Page'; +import ItemGroup from 'src/debug/ItemGroup'; +import ItemMerger from 'src/debug/ItemMerger'; +import { items } from 'test/testItems'; +import ChangeTracker from 'src/debug/ChangeTracker'; + +test('empty', async () => { + const tracker = new ChangeTracker(); + expect(asPages(tracker, [])).toEqual([]); +}); + +test('no merger', async () => { + const pageItems = [ + items(0, [{ id: 1, line: 1 }]), + items(1, [ + { id: 2, line: 1 }, + { id: 3, line: 1 }, + { id: 4, line: 1 }, + ]), + items(2, [{ id: 5, line: 1 }]), + ]; + const flattenedItems = new Array().concat(...pageItems); + const tracker = new ChangeTracker(); + const pages = asPages(tracker, flattenedItems); + expect(pages).toEqual([ + { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) }, + { index: 1, itemGroups: pageItems[1].map((item) => new ItemGroup(item)) }, + { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) }, + ] as Page[]); + expect(tracker.changeCount()).toEqual(0); +}); + +test('merger', async () => { + const pageItems = [ + items(0, [{ id: 1, line: 1 }]), + items(1, [ + { id: 2, line: 1 }, + { id: 3, line: 1 }, + { id: 4, line: 2 }, + ]), + items(2, [{ id: 5, line: 1 }]), + ]; + const flattenedItems = new Array().concat(...pageItems); + const merger: ItemMerger = { groupKey: 'line', merge: (items) => items[0] }; + const tracker = new ChangeTracker(); + const pages = asPages(tracker, flattenedItems, merger); + + expect(pages).toEqual([ + { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) }, + { + index: 1, + itemGroups: [ + new ItemGroup(merger.merge(tracker, pageItems[1].slice(0, 2)), pageItems[1].slice(0, 2)), + new ItemGroup(pageItems[1][2]), + ], + }, + { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) }, + ] as Page[]); +}); diff --git a/core/test/debug/detectChanges.test.ts b/core/test/debug/detectChanges.test.ts index 3dc8b2a..f3dd5ec 100644 --- a/core/test/debug/detectChanges.test.ts +++ b/core/test/debug/detectChanges.test.ts @@ -1,4 +1,6 @@ -import { detectChanges, PositionChange, Direction } from 'src/debug/detectChanges'; +import ChangeTracker from 'src/debug/ChangeTracker'; +import { detectChanges } from 'src/debug/detectChanges'; +import { PositionChange, Direction } from 'src/debug/ChangeIndex'; import { items } from 'test/testItems'; test('No changes', async () => { @@ -9,8 +11,9 @@ test('No changes', async () => { { id: 'B', line: '1', x: 2 }, ]); - const changes = detectChanges(inputItems, inputItems); - expect(changes).toEqual(new Map()); + const tracker = new ChangeTracker(); + detectChanges(tracker, inputItems, inputItems); + expect(tracker.changeCount()).toEqual(0); }); test('Positional changes', async () => { const inputItems = items(0, [ @@ -22,11 +25,9 @@ test('Positional changes', async () => { const transformedItems = [inputItems[0], inputItems[3], inputItems[2], inputItems[1]]; - const changes = detectChanges(inputItems, transformedItems); - expect(changes).toEqual( - new Map([ - [inputItems[1].uuid, new PositionChange(Direction.DOWN, 2)], - [inputItems[3].uuid, new PositionChange(Direction.UP, 2)], - ]), - ); + const tracker = new ChangeTracker(); + const changes = detectChanges(tracker, inputItems, transformedItems); + expect(tracker.changeCount()).toEqual(2); + expect(tracker.change(inputItems[1])).toEqual(new PositionChange(Direction.DOWN, 2)); + expect(tracker.change(inputItems[3])).toEqual(new PositionChange(Direction.UP, 2)); }); diff --git a/core/test/support/itemUtils.test.ts b/core/test/support/groupingUtils.test.ts similarity index 62% rename from core/test/support/itemUtils.test.ts rename to core/test/support/groupingUtils.test.ts index 644160c..8544911 100644 --- a/core/test/support/itemUtils.test.ts +++ b/core/test/support/groupingUtils.test.ts @@ -1,14 +1,10 @@ import Item from 'src/Item'; -import Page from 'src/support/Page'; import { groupByPage, groupByElement, transformGroupedByPage, transformGroupedByPageAndLine, - asPages, -} from 'src/support/itemUtils'; -import ItemGroup from 'src/support/ItemGroup'; -import ItemMerger from 'src/support/ItemMerger'; +} from 'src/support/groupingUtils'; import { items } from 'test/testItems'; describe('groupByPage', () => { @@ -97,46 +93,3 @@ describe('transformGroupedByPageAndLine', () => { expect(transformedItems.map((item) => item.data['group'])).toEqual(['0/1:1', '1/1:2', '1/2:1', '2/1:1']); }); }); - -describe('asPages', () => { - test('empty', async () => { - expect(groupByPage([])).toEqual([]); - }); - - test('no merger', async () => { - const pageItems = [ - [new Item(0, { id: 1, line: 1 })], - [new Item(1, { id: 2, line: 1 }), new Item(1, { id: 3, line: 1 }), new Item(1, { id: 4, line: 2 })], - [new Item(2, { id: 5, line: 1 })], - ]; - const flattenedItems = new Array().concat(...pageItems); - const pages = asPages(flattenedItems); - expect(pages).toEqual([ - { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) }, - { index: 1, itemGroups: pageItems[1].map((item) => new ItemGroup(item)) }, - { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) }, - ] as Page[]); - }); - - test('merger', async () => { - const pageItems = [ - [new Item(0, { id: 1, line: 1 })], - [new Item(1, { id: 2, line: 1 }), new Item(1, { id: 3, line: 1 }), new Item(1, { id: 4, line: 2 })], - [new Item(2, { id: 5, line: 1 })], - ]; - const flattenedItems = new Array().concat(...pageItems); - const merger: ItemMerger = { groupKey: 'line', merge: (items) => items[0] }; - const pages = asPages(flattenedItems, merger); - expect(pages).toEqual([ - { index: 0, itemGroups: pageItems[0].map((item) => new ItemGroup(item)) }, - { - index: 1, - itemGroups: [ - new ItemGroup(merger.merge(pageItems[1].slice(0, 2)), pageItems[1].slice(0, 2)), - new ItemGroup(pageItems[1][2]), - ], - }, - { index: 2, itemGroups: pageItems[2].map((item) => new ItemGroup(item)) }, - ] as Page[]); - }); -}); diff --git a/core/test/testItems.ts b/core/test/testItems.ts index bf8fade..45d213e 100644 --- a/core/test/testItems.ts +++ b/core/test/testItems.ts @@ -3,3 +3,20 @@ import Item from 'src/Item'; export function items(page: number, data: object[]): Item[] { return data.map((data) => new Item(page, data)); } + +export function realisticItems(page: number, data: object[]): Item[] { + return data.map( + (data, idx) => + new Item(page, { + line: idx, + x: idx, + y: idx, + str: `text ${idx}`, + fontName: 'g_d0_f2', + dir: 'ltr', + width: 10, + height: 10, + ...data, + }), + ); +} diff --git a/ui/src/components/ComponentDefinition.ts b/ui/src/components/ComponentDefinition.ts new file mode 100644 index 0000000..262ba56 --- /dev/null +++ b/ui/src/components/ComponentDefinition.ts @@ -0,0 +1,5 @@ +import type { SvelteComponent } from 'svelte'; + +export default class ComponentDefinition { + constructor(public component: object, public args: object = {}) {} +} diff --git a/ui/src/debug/ChangeSymbol.svelte b/ui/src/debug/ChangeSymbol.svelte index 598c448..9cb040a 100644 --- a/ui/src/debug/ChangeSymbol.svelte +++ b/ui/src/debug/ChangeSymbol.svelte @@ -1,25 +1,40 @@ -{#if changeContent} +{#if hasChanged}
- {#if icon} - + {#if iconComp} + + {/if} + {#if changeContent} +
{changeContent}
{/if} -
{changeContent}
{/if} diff --git a/ui/src/debug/DebugView.svelte b/ui/src/debug/DebugView.svelte index f63bfa3..54f93d1 100644 --- a/ui/src/debug/DebugView.svelte +++ b/ui/src/debug/DebugView.svelte @@ -5,7 +5,6 @@ import { BookOpen, ArrowLeft, ArrowRight } from 'svelte-hero-icons'; import type Debugger from '@core/Debugger'; - import { asPages } from '../../../core/src/support/itemUtils'; import Popup from '../components/Popup.svelte'; import PageSelectionPopup from './PageSelectionPopup.svelte'; @@ -24,10 +23,9 @@ $: canPrev = currentStage > 0; $: stageResult = debug.stageResults(currentStage); $: pageIsPinned = !isNaN(pinnedPage); - $: pagesNumbers = new Set(stageResult.items.map((item) => item.page)); - $: pages = asPages(stageResult.items, stageResult.descriptor?.debug?.itemMerger); + $: pagesNumbers = new Set(stageResult.pages.map((page) => page.index)); $: maxPage = Math.max(...pagesNumbers); - $: visiblePages = pageIsPinned ? pages.filter((page) => page.index === pinnedPage) : pages; + $: visiblePages = pageIsPinned ? stageResult.pages.filter((page) => page.index === pinnedPage) : stageResult.pages;
diff --git a/ui/src/debug/ItemTable.svelte b/ui/src/debug/ItemTable.svelte index 2a041d7..414104c 100644 --- a/ui/src/debug/ItemTable.svelte +++ b/ui/src/debug/ItemTable.svelte @@ -2,11 +2,10 @@ import { scale, fade } from 'svelte/transition'; import type AnnotatedColumn from '@core/debug/AnnotatedColumn'; import ColumnAnnotation from '../../../core/src/debug/ColumnAnnotation'; - import type { Change } from '@core/debug/detectChanges'; - import { ChangeCategory } from '../../../core/src/debug/detectChanges'; + import type ChangeIndex from '@core/debug/ChangeIndex'; import inView from '../actions/inView'; import { formatValue } from './formatValues'; - import type Page from '@core/support/Page'; + import type Page from '@core/debug/Page'; import ChangeSymbol from './ChangeSymbol.svelte'; export let schema: AnnotatedColumn[]; @@ -14,7 +13,7 @@ export let maxPage: number; export let pageIsPinned: boolean; export let onlyRelevantItems: boolean; - export let changes: Map; + export let changes: ChangeIndex; let maxItemsToRenderInOneLoad = 200; let renderedMaxPage = 0; let expandedItemGroup: { pageIndex: number; itemIndex: number }; @@ -75,17 +74,17 @@ {/if} - {#each page.itemGroups.filter((group) => !onlyRelevantItems || changes.has(group.top.uuid)) as itemGroup, itemIdx} + {#each page.itemGroups.filter((group) => !onlyRelevantItems || changes.hasChanged(group.top)) as itemGroup, itemIdx} {#if itemIdx === 0} - +
Page {page.index} {pageIsPinned ? '' : ' / ' + maxPage}
{:else} @@ -93,9 +92,11 @@ {/if} itemGroup.hasMany() && toggleRow(page.index, itemIdx)}> - -
{itemIdx}{itemGroup.hasMany() ? '+' : ''}
- + +
+ +
{itemIdx}{itemGroup.hasMany() ? '…' : ''}
+
@@ -108,9 +109,18 @@ {#if expandedItemGroup && isExpanded(page.index, itemIdx)} {#each itemGroup.elements as child, childIdx} - + - {'└ ' + childIdx} + +
+
{'└ ' + childIdx}
+ +
+ {#each schema as column} {formatValue(child.data[column.name])} {/each} @@ -122,7 +132,7 @@ -{#if onlyRelevantItems && changes.size === 0} +{#if onlyRelevantItems && changes.changeCount() === 0}
No changes from the transformation.
Want to see
@@ -144,13 +154,12 @@