extern/pdf-to-markdown
1
0
Fork 0
mirror of https://github.com/jzillmann/pdf-to-markdown.git synced 2025-06-25 12:01:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

Split result files

Browse Source 
* Due 100 MB limit of Github
This commit is contained in:
Johannes Zillmann 2021-03-13 22:46:10 +01:00
parent 713a82b41d
commit d5523fb1d4
1 changed files with 37 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
core/test/Files.test.ts
View File

@ -26,21 +26,33 @@ describe.each(files)('Test %p', (file) => {
test.each(transformers.map((t) => t.name))('stage %p', (transformerName) => { test.each(transformers.map((t) => t.name))('stage %p', (transformerName) => {
const stageResults = debug.stageResults(debug.stageNames.indexOf(transformerName)); const stageResults = debug.stageResults(debug.stageNames.indexOf(transformerName));
const lines: string[] = [];
let groupedItemCount = stageResults const chunkedLines: string[][] = [[]];
.selectPages(false, true) let resultIndex = 0;
.reduce((itemCount, page) => itemCount + page.itemGroups.length, 0); let collectedItems = 0;
stageResults.selectPages(true, true).forEach((page) => { stageResults.selectPages(true, true).forEach((page) => {
page.itemGroups.forEach((itemGroup) => { page.itemGroups.forEach((itemGroup) => {
const change = stageResults.changes.change(itemGroup.top); const change = stageResults.changes.change(itemGroup.top);
if (change || stageResults.descriptor.debug?.showAll) { if (change || stageResults.descriptor.debug?.showAll) {
const item = itemGroup.top; const item = itemGroup.top;
const changeType = change?.constructor.name || 'none'; const changeType = change?.constructor.name || 'none';
lines.push(itemToString(debug.fontMap, item, changeType)); chunkedLines[resultIndex].push(itemToString(debug.fontMap, item, changeType));
collectedItems++;
} }
}); });
// we split results to multiple files to circumvent githubs 100MB limit
resultIndex = Math.floor(collectedItems / 450_000);
if (resultIndex === chunkedLines.length) {
chunkedLines.push([]);
}
}); });
lines.unshift(
// Global characteristics
let groupedItemCount = stageResults
.selectPages(false, true)
.reduce((itemCount, page) => itemCount + page.itemGroups.length, 0);
chunkedLines[0].unshift(
JSON.stringify( JSON.stringify(
{ {
pages: stageResults.pages.length, pages: stageResults.pages.length,
@ -54,13 +66,16 @@ describe.each(files)('Test %p', (file) => {
2, 2,
), ),
); );
chunkedLines.forEach((lines, idx) => {
const transformerResultAsString = lines.join('\n') || '{}'; const transformerResultAsString = lines.join('\n') || '{}';
const resultFolder = `${folder}/${file.substr(0, file.length - 4)}`; const resultFolder = `${folder}/${file.substr(0, file.length - 4)}`;
const resultFile = `${resultFolder}/${ const fileName = `${transformerName[0].toLowerCase() + transformerName.slice(1).replace(/\s/g, '')}`;
transformerName[0].toLowerCase() + transformerName.slice(1).replace(/\s/g, '') const fileIndex = chunkedLines.length > 1 ? `.${idx}` : '';
}.json`; const resultFile = `${resultFolder}/${fileName}${fileIndex}.json`;
expect(transformerResultAsString).toMatchFile(resultFile); expect(transformerResultAsString).toMatchFile(resultFile);
}); });
});
}); });
function itemToString(fontMap: Map<string, object>, item: Item, changeType: string): string { function itemToString(fontMap: Map<string, object>, item: Item, changeType: string): string {
@ -73,9 +88,18 @@ function itemToString(fontMap: Map<string, object>, item: Item, changeType: stri
newFontName = fontName.map((name) => fontMap.get(name)?.['name']); newFontName = fontName.map((name) => fontMap.get(name)?.['name']);
} }
} }
const transform: undefined | number[] = item.data['transform'];
let newTransform;
if (transform) {
newTransform = transform.map((num) => num.toFixed(2));
}
return JSON.stringify({ return JSON.stringify({
page: item.page, page: item.page,
change: changeType, change: changeType,
data: { ...item.data, fontName: newFontName }, ...item.data,
fontName: newFontName,
height: item.data['height'].toFixed(2),
width: item.data['width'].toFixed(2),
transform: newTransform,
}); });
} }