WebScraper/Scraper.js at master · Ciaran97/WebScraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
var puppeteer = require('puppeteer');
const Excel = require('exceljs');

var pad = "000000";


async function scrapeProduct() {

    const workbook = new Excel.Workbook();
    const worksheet = workbook.addWorksheet("Branch Codes");

    worksheet.columns = [
        {header: 'Branch Code', key: 'branchcode', width: 10},
        {header: 'Swift Code', key: 'bic', width: 32},
        {header: 'Bank Name', key: 'bankName', width: 15,},
        {header: 'Branch Name', key: 'branchName', width: 15,},
        {header: 'Branch Address', key: 'branchAddress', width: 15,}
      ];


try{
 const browser = await puppeteer.launch();
    const page = await browser.newPage();
for(i = 18005; i <= 999999; i++){
//var code =
let code = "" + i;

//Padding for branch codes
var bc = pad.substring(0, pad.length - i.toString().length) + i

//Website to scrape
var url = "https://www.iban.co.za/branch-code-" + bc + ".html"


    await page.goto(url);


    let texts = await page.evaluate(() => {

//Set Html attribute to read here
       const tds = Array.from(document.querySelectorAll('table tr td'))
       return tds.map(td => td.innerText)

    });
console.log(i);
    if(texts[1]){
        worksheet.addRow({branchcode: texts[1], bic: texts[3], bankName: texts[5], branchName: texts[7], branchAddress: texts[9]});

        console.log("Row added");
    }else{
            console.log("No Data");
    }

/*
    const [el] = await page.$x('//*[@id="wrap"]/div[2]/div/div[1]/table');
    const src = await el.getProperty('src');
    const srcText = await src.jsonValue();

    console.log({srcText});

    */

}

browser.close();
}
catch(e){
    console.log(e, "ERROR");
}
await workbook.xlsx.writeFile('export.xlsx');
}


scrapeProduct();


/*


puppeteer.launch().then(browser => browser.newPage()
.then(page => {return page.goto(url).then(function(){
    return page.content()
})})).then(html => {
    const $ = cheerio.load(html);
    const newsHeadlines = [];
    $('//*[@id="wrap"]/div[2]/div/div[1]/h1').each(function() {
      newsHeadlines.push({
        title: $(this).text(),
      });
    });

    console.log(newsHeadlines);
  })
  .catch(console.error);


*/

async function exTest(){
    const workbook = new Excel.Workbook();
    const worksheet = workbook.addWorksheet("Branch Codes");

  worksheet.columns = [
    {header: 'Branch Code', key: 'branchcode', width: 10},
    {header: 'Swift Code', key: 'bic', width: 32},
    {header: 'Bank Name', key: 'bankName', width: 15,},
    {header: 'Branch Name', key: 'branchName', width: 15,},
    {header: 'Branch Address', key: 'branchAddress', width: 15,}
  ];

  worksheet.addRow({branchcode: 1, bic: 'John Doe', bankName: new Date(1970, 1, 1)});
  worksheet.addRow({branchcode: 2, bic: 'Jane Doe', bankName: new Date(1965, 1, 7)});

  // save under export.xlsx
  await workbook.xlsx.writeFile('export.xlsx');

  // load a copy of export.xlsx
  const newWorkbook = new Excel.Workbook();
  await newWorkbook.xlsx.readFile('export.xlsx');

  const newworksheet = newWorkbook.getWorksheet('My Sheet');
  newworksheet.addRow(
    {branchcode: 3, bic: 'New Guy', bankName: new Date(2000, 1, 1)}
  );

  await newWorkbook.xlsx.writeFile('export2.xlsx');

  console.log("File is written");
  };

 // exTest();