-
Notifications
You must be signed in to change notification settings - Fork 14
/
dumpUser.js
121 lines (106 loc) · 2.58 KB
/
dumpUser.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import fs from 'fs';
import crypto from 'crypto';
import elasticsearch from '@elastic/elasticsearch';
import csvStringify from 'csv-stringify';
import JSZip from 'jszip';
const ELASTICSEARCH_URL = 'http://localhost:62223';
const OUTPUT_DIR = './data';
const client = new elasticsearch.Client({
node: ELASTICSEARCH_URL,
});
/**
* @param {any[][]} input
* @returns {Promise<string>} CSV content
*/
function generateCSV(input) {
return new Promise((resolve, reject) => {
csvStringify(input, (err, csvData) => {
if (err) {
return reject(err);
}
return resolve(csvData);
});
});
}
/**
* @param {string} input
* @returns {string} - input's sha256 hash hex string. Empty string if input is falsy.
*/
function sha256(input) {
return input
? crypto.createHash('sha256').update(input, 'utf8').digest('hex')
: '';
}
async function scanIndex(index) {
let result = [];
const { body: initialResult } = await client.search({
index,
size: 200,
scroll: '5m',
});
const totalCount = initialResult.hits.total;
initialResult.hits.hits.forEach((hit) => {
result.push(hit);
});
while (result.length < totalCount) {
const { body: scrollResult } = await client.scroll({
scrollId: initialResult._scroll_id,
scroll: '5m',
});
scrollResult.hits.hits.forEach((hit) => {
result.push(hit);
});
}
return result;
}
/**
* @param {object[]} articles
* @returns {Promise<string>} Generated CSV string
*/
function dumpUsers(users) {
return generateCSV([
[
'userIdsha256',
'name',
'email',
'facebookId',
'githubId',
'twitterId',
'updatedAt',
],
...users.map(({ _id, _source }) => [
sha256(_id),
_source.name,
_source.email,
_source.facebookId,
_source.githubId,
_source.twitterId,
_source.updatedAt,
]),
]);
}
/**
* @param {string} fileName The name of file to be put in a zip file
* @returns {({string}) => (none)}
*/
function writeFile(fileName) {
return (data) => {
const zip = new JSZip();
zip.file(fileName, data);
// Ref: https://stuk.github.io/jszip/documentation/howto/write_zip.html#in-nodejs
//
zip
.generateNodeStream({
type: 'nodebuffer',
streamFiles: true,
compression: 'DEFLATE',
compressionOptions: { level: 8 },
})
.pipe(fs.createWriteStream(`${OUTPUT_DIR}/${fileName}.zip`))
.on('finish', () => console.log(`${fileName}.zip written.`));
};
}
/**
* Main process
*/
scanIndex('users').then(dumpUsers).then(writeFile('users.csv'));