Skip to content

[WIP] Replace sax with saxes #1127

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Feb 20, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions lib/stream/xlsx/hyperlink-reader.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const {EventEmitter} = require('events');
const Sax = require('sax');
const SAXStream = require('../../utils/sax-stream');

const Enums = require('../../doc/enums');
const RelType = require('../../xlsx/rel-type');
Expand Down Expand Up @@ -42,8 +42,8 @@ class HyperlinkReader extends EventEmitter {
return;
}

const parser = Sax.createStream(true, {});
parser.on('opentag', node => {
const saxStream = new SAXStream();
saxStream.sax.on('opentag', node => {
if (node.name === 'Relationship') {
const rId = node.attributes.Id;
switch (node.attributes.Type) {
Expand All @@ -69,13 +69,13 @@ class HyperlinkReader extends EventEmitter {
}
});

parser.on('end', () => {
saxStream.sax.on('end', () => {
this.emit('finished');
});

// create a down-stream flow-control to regulate the stream
const flowControl = this._workbook.flowControl.createChild();
flowControl.pipe(parser, {sync: true});
flowControl.pipe(saxStream, {sync: true});
entry.pipe(flowControl);
}
}
Expand Down
16 changes: 8 additions & 8 deletions lib/stream/xlsx/workbook-reader.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ const fs = require('fs');
const {EventEmitter} = require('events');
const Stream = require('stream');
const unzip = require('unzipper');
const Sax = require('sax');
const tmp = require('tmp');
const SAXStream = require('../../utils/sax-stream');

const FlowControl = require('../../utils/flow-control');

Expand Down Expand Up @@ -206,18 +206,18 @@ class WorkbookReader extends EventEmitter {
return;
}

const parser = Sax.createStream(true, {});
const saxStream = new SAXStream();
let inT = false;
let t = null;
let index = 0;
parser.on('opentag', node => {
saxStream.sax.on('opentag', node => {
if (node.name === 't') {
t = null;
inT = true;
}
});
parser.on('closetag', name => {
if (inT && name === 't') {
saxStream.sax.on('closetag', node => {
if (inT && node.name === 't') {
if (sharedStrings) {
sharedStrings.push(t);
} else {
Expand All @@ -226,13 +226,13 @@ class WorkbookReader extends EventEmitter {
t = null;
}
});
parser.on('text', text => {
saxStream.sax.on('text', text => {
t = t ? t + text : text;
});
parser.on('error', error => {
saxStream.sax.on('error', error => {
this.emit('error', error);
});
entry.pipe(parser);
entry.pipe(saxStream);
}

_parseStyles(entry, options) {
Expand Down
24 changes: 14 additions & 10 deletions lib/stream/xlsx/worksheet-reader.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const {EventEmitter} = require('events');
const Sax = require('sax');
const SAXStream = require('../../utils/sax-stream');

const _ = require('../../utils/under-dash');
const utils = require('../../utils/utils');
Expand Down Expand Up @@ -139,8 +139,8 @@ class WorksheetReader extends EventEmitter {
let c = null;
let current = null;

const parser = Sax.createStream(true, {});
parser.on('opentag', node => {
const saxStream = new SAXStream();
saxStream.sax.on('opentag', node => {
if (emitSheet) {
switch (node.name) {
case 'cols':
Expand Down Expand Up @@ -231,17 +231,20 @@ class WorksheetReader extends EventEmitter {
});

// only text data is for sheet values
parser.on('text', text => {
saxStream.sax.on('text', text => {
if (emitSheet) {
if (current) {
current.text += text;
}
}
});

parser.on('closetag', name => {
saxStream.sax.on('closetag', node => {
if (emitSheet) {
switch (name) {
switch (node.name) {
case 'worksheet':
saxStream.sax.close();
break;
case 'cols':
inCols = false;
this._columns = Column.fromModel(cols);
Expand Down Expand Up @@ -330,7 +333,7 @@ class WorksheetReader extends EventEmitter {
}
}
if (emitHyperlinks || hyperlinks) {
switch (name) {
switch (node.name) {
case 'hyperlinks':
inHyperlinks = false;
break;
Expand All @@ -339,16 +342,17 @@ class WorksheetReader extends EventEmitter {
}
}
});
parser.on('error', error => {
saxStream.sax.on('error', error => {
this.emit('error', error);
});
parser.on('end', () => {
saxStream.sax.on('end', () => {
this.emit('finished');
});

// create a down-stream flow-control to regulate the stream
const flowControl = this.workbook.flowControl.createChild();
flowControl.pipe(parser, {sync: true});

flowControl.pipe(saxStream, {sync: true});
entry.pipe(flowControl);
}
}
Expand Down
14 changes: 14 additions & 0 deletions lib/utils/sax-stream.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
const {Writable} = require('stream');
const saxes = require('saxes');

module.exports = class SAXStream extends Writable {
constructor() {
super();
this.sax = new saxes.SaxesParser();
}

_write(chunk, _enc, cb) {
this.sax.write(chunk.toString());
if (typeof cb === 'function') cb();
}
};
38 changes: 20 additions & 18 deletions lib/xlsx/xform/base-xform.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
const Sax = require('sax');

const SAXStream = require('../../utils/sax-stream');
const XmlStream = require('../../utils/xml-stream');

/* 'virtual' methods used as a form of documentation */
Expand Down Expand Up @@ -57,55 +56,58 @@ class BaseXform {
this.model = Object.assign(this.model || {}, obj);
}

parse(parser, stream) {
parse(saxStream, stream) {
return new Promise((resolve, reject) => {
const abort = error => {
// Abandon ship! Prevent the parser from consuming any more resources
parser.removeAllListeners();
parser.on('error', () => {}); // Ignore any parse errors from the chunk being processed
stream.unpipe(parser);
saxStream.sax.off('opentag');
saxStream.sax.off('text');
saxStream.sax.off('closetag');
saxStream.sax.off('error');
saxStream.sax.off('end');
saxStream.sax.on('error', () => {}); // Ignore any parse errors from the chunk being processed
if (stream) {
stream.unpipe(saxStream);
}
reject(error);
};

parser.on('opentag', node => {
saxStream.sax.on('opentag', node => {
try {
// console.log('opentag', node.name);
this.parseOpen(node);
} catch (error) {
abort(error);
}
});
parser.on('text', text => {
saxStream.sax.on('text', text => {
try {
this.parseText(text);
} catch (error) {
abort(error);
}
});
parser.on('closetag', name => {
saxStream.sax.on('closetag', node => {
try {
// console.log('closetag', name);
if (!this.parseClose(name)) {
if (!this.parseClose(node.name)) {
resolve(this.model);
}
} catch (error) {
abort(error);
}
});
parser.on('end', () => {
// console.log('end');
saxStream.sax.on('end', () => {
resolve(this.model);
});
parser.on('error', error => {
saxStream.sax.on('error', error => {
abort(error);
});
});
}

parseStream(stream) {
const parser = Sax.createStream(true, {});
const promise = this.parse(parser, stream);
stream.pipe(parser);
const saxStream = new SAXStream();
const promise = this.parse(saxStream, stream);
stream.pipe(saxStream);

return promise;
}
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@
"fast-csv": "^3.4.0",
"jszip": "^3.1.5",
"proxyquire": "^2.1.3",
"sax": "^1.2.4",
"saxes": "5.0.0-rc.2",
"tmp": "^0.1.0",
"unzipper": "^0.9.12",
"uuid": "^3.3.3"
Expand Down
6 changes: 3 additions & 3 deletions spec/integration/pr/pr-896/test-pr-896.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ const {expect} = chai;

const TEST_XLSX_FILE_NAME = './spec/out/wb.test.xlsx';
const RT_ARR = [
{text: 'First Line:\r\n', font: {bold: true}},
{text: 'Second Line\r\n'},
{text: 'Third Line\r\n'},
{text: 'First Line:\n', font: {bold: true}},
{text: 'Second Line\n'},
{text: 'Third Line\n'},
{text: 'Last Line'},
];
const TEST_VALUE = {
Expand Down
2 changes: 1 addition & 1 deletion spec/integration/workbook-xlsx-reader.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ describe('WorkbookReader', () => {
},
err => {
expect(err.message).to.equal(
'Text data outside of root node.\nLine: 1\nColumn: 1\nChar: f'
'3:1: text data outside of root node.'
);
// Wait a tick before checking for an unhandled rejection
return new Promise(setImmediate);
Expand Down
15 changes: 7 additions & 8 deletions spec/unit/xlsx/xform/test-xform-helper.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
const Sax = require('sax');
const {cloneDeep, each} = require('../../../utils/under-dash');
const CompyXform = require('./compy-xform');

const SAXStream = verquire('utils/sax-stream');
const XmlStream = verquire('utils/xml-stream');
const BooleanXform = verquire('xlsx/xform/simple/boolean-xform');

Expand Down Expand Up @@ -130,10 +130,9 @@ const its = {
},
],
});
const parser = Sax.createStream(true);

const saxStream = new SAXStream();
xform
.parse(parser)
.parse(saxStream)
.then(model => {
// console.log('parsed Model', JSON.stringify(model));
// console.log('expected Model', JSON.stringify(result));
Expand All @@ -147,7 +146,7 @@ const its = {
resolve();
})
.catch(reject);
parser.write(xml);
saxStream.write(xml);
}));
},

Expand All @@ -157,11 +156,11 @@ const its = {
const xml = getExpectation(expectation, 'xml');
const result = getExpectation(expectation, 'parsedModel');

const parser = Sax.createStream(true);
const saxStream = new SAXStream();
const xform = expectation.create();

xform
.parse(parser)
.parse(saxStream)
.then(model => {
// eliminate the undefined
const clone = cloneDeep(model, false);
Expand All @@ -173,7 +172,7 @@ const its = {
})
.catch(reject);

parser.write(xml);
saxStream.write(xml);
}));
},

Expand Down