daim/src/process/4-enrich-docker-compose.process.js at main · DatabaseEvolutionNudgeInMicroservices/daim · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
// IMPORTS

import { Env } from '../helper/env.helper.js';
import { logger } from '../helper/logger.helper.js';
import { MongoDB } from '../dao/mongodb.dao.js';
import { Process } from './process.process.js';
import { GitHubApiRequest } from '../model/github-api-request.model.js';
import { GitHubApiClient } from '../helper/github-api-client.helper.js';
import { GitHubApiQueue } from '../helper/github-api-queue.helper.js';

/**
 * Represents a process to enrich the repositories data.
 */
class ProcessEnrichDockerCompose extends Process {
  constructor() {
    super();

    // Dependencies.
    this.env = new Env(); // Environment variables.
    this.mongoDb = new MongoDB(
      this.env.getMongoDbUrl(),
      this.env.getMongoDbName(),
    ); // MongoDB connection.
    this.githubApiClients = this.env
      .getGitHubApiTokens()
      .map((token) => new GitHubApiClient(token)); // GitHub search API clients.
    this.githubApiQueue = new GitHubApiQueue(this.githubApiClients); // GitHub search API queue.

    // Working variables.
    this.repositoryTotalCount = 0;
    this.repositoryCurrentCount = 0;
  }

  /**
   * Executes the process to enrich repositories.
   */
  process() {
    this.githubApiQueue.start();

    const enrich = async () => {
      // Retrieve repositories.
      const cursor = await this.mongoDb.getRepositories('repositories');
      while (await cursor.hasNext()) {
        let repository = await cursor.next();
        // ---
        // ENRICH: Docker-Compose files
        // ---
        let { docker_compose_files } = repository;
        for (let i = 0; i < docker_compose_files.length; i++) {
          const file = docker_compose_files[i];
          if (file) {
            const url = `https://api.github.com/repos/${repository._id}/contents/${encodeURIComponent(file)}`;
            this.githubApiQueue.push(
              new GitHubApiRequest(url, {}, (result) => {
                if (result && result.data) {
                  let content = '';
                  if (result.data.content) {
                    content = Buffer.from(
                      result.data.content,
                      'base64',
                    ).toString('utf-8');
                  }
                  if (!repository.docker_compose_files_content[i]) {
                    repository.docker_compose_files_content[i] = content;
                    this.mongoDb.saveRepository(
                      repository._id,
                      repository.url,
                      repository.branch,
                      repository.owner,
                      repository.title,
                      repository.description,
                      repository.topics,
                      repository.creation_date,
                      repository.last_updated_date,
                      repository.size,
                      repository.stars_count,
                      repository.commits_count,
                      repository.contributors_count,
                      repository.folders_count,
                      repository.services_directories,
                      repository.services_files,
                      repository.readme_files,
                      repository.readme_files_content,
                      repository.docker_compose_files,
                      repository.docker_compose_files_content,
                      repository.istio_files,
                      repository.istio_files_content,
                      -1,
                      [],
                      repository.languages,
                      'repositories',
                    );
                  }
                  logger.info(
                    `[enrich-docker-compose] Repository: repository_id=${repository._id}, file=${file}`,
                  );
                }
              }),
            );
          }
        }
      }
    };

    this.mongoDb
      .connect()
      .then(() => {
        enrich(); // Process entry point.
      })
      .catch((error) => {
        logger.error(`[enrich-docker-compose] ${error.message}`);
      });
  }
}

let processEnrichDockerCompose = new ProcessEnrichDockerCompose();
processEnrichDockerCompose.process();