2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// Imports
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								import * as fs from 'fs';
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								import * as path from 'path';
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								import * as shell from 'shelljs';
							 | 
						
					
						
							
								
									
										
										
										
											2017-06-25 01:40:04 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								import {HIDDEN_DIR_PREFIX} from '../common/constants';
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								import {GithubApi} from '../common/github-api';
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								import {GithubPullRequests} from '../common/github-pull-requests';
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-27 18:07:25 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								import {assertNotMissingOrEmpty, getPrInfoFromDownloadPath, Logger} from '../common/utils';
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								// Classes
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								export class BuildCleaner {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-27 18:07:25 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  private logger = new Logger('BuildCleaner');
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-12 15:39:16 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  // Constructor
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  constructor(protected buildsDir: string, protected githubOrg: string, protected githubRepo: string,
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								              protected githubToken: string, protected downloadsDir: string, protected artifactPath: string) {
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-27 21:04:43 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    assertNotMissingOrEmpty('buildsDir', buildsDir);
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    assertNotMissingOrEmpty('githubOrg', githubOrg);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    assertNotMissingOrEmpty('githubRepo', githubRepo);
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-27 22:40:13 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    assertNotMissingOrEmpty('githubToken', githubToken);
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    assertNotMissingOrEmpty('downloadsDir', downloadsDir);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    assertNotMissingOrEmpty('artifactPath', artifactPath);
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  // Methods - Public
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public async cleanUp(): Promise<void> {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    try {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      this.logger.log('Cleaning up builds and downloads');
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      const openPrs = await this.getOpenPrNumbers();
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      this.logger.log(`Open pull requests: ${openPrs.length}`);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      await Promise.all([
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        this.cleanBuilds(openPrs),
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        this.cleanDownloads(openPrs),
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      ]);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    } catch (error) {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      this.logger.error('ERROR:', error);
							 | 
						
					
						
							
								
									
										
										
										
											2020-05-02 16:17:48 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								      throw error;
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public async cleanBuilds(openPrs: number[]): Promise<void> {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    const existingBuilds = await this.getExistingBuildNumbers();
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    await this.removeUnnecessaryBuilds(existingBuilds, openPrs);
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public async cleanDownloads(openPrs: number[]): Promise<void> {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    const existingDownloads = await this.getExistingDownloads();
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    await this.removeUnnecessaryDownloads(existingDownloads, openPrs);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public getExistingBuildNumbers(): Promise<number[]> {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    return new Promise<number[]>((resolve, reject) => {
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      fs.readdir(this.buildsDir, (err, files) => {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        if (err) {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								          return reject(err);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        const buildNumbers = files.
							 | 
						
					
						
							
								
									
										
										
										
											2017-06-25 01:40:04 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								          map(name => name.replace(HIDDEN_DIR_PREFIX, '')).   // Remove the "hidden dir" prefix
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								          map(Number).                                        // Convert string to number
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								          filter(Boolean);                                    // Ignore NaN (or 0), because they are not builds
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        resolve(buildNumbers);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      });
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    });
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public async getOpenPrNumbers(): Promise<number[]> {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    const api = new GithubApi(this.githubToken);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    const githubPullRequests = new GithubPullRequests(api, this.githubOrg, this.githubRepo);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    const prs = await githubPullRequests.fetchAll('open');
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    return prs.map(pr => pr.number);
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public removeDir(dir: string): void {
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    try {
							 | 
						
					
						
							
								
									
										
										
										
											2017-06-25 01:40:04 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								      if (shell.test('-d', dir)) {
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-08 12:55:15 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								        shell.chmod('-R', 'a+w', dir);
							 | 
						
					
						
							
								
									
										
										
										
											2017-06-25 01:40:04 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								        shell.rm('-rf', dir);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      }
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    } catch (err) {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-12 15:39:16 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								      this.logger.error(`ERROR: Unable to remove '${dir}' due to:`, err);
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public removeUnnecessaryBuilds(existingBuildNumbers: number[], openPrNumbers: number[]): void {
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    const toRemove = existingBuildNumbers.filter(num => !openPrNumbers.includes(num));
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-12 15:39:16 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    this.logger.log(`Existing builds: ${existingBuildNumbers.length}`);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    this.logger.log(`Removing ${toRemove.length} build(s): ${toRemove.join(', ')}`);
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2017-06-25 01:40:04 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    // Try removing public dirs.
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    toRemove.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      map(num => path.join(this.buildsDir, String(num))).
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      forEach(dir => this.removeDir(dir));
							 | 
						
					
						
							
								
									
										
										
										
											2017-06-25 01:40:04 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    // Try removing hidden dirs.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    toRemove.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      map(num => path.join(this.buildsDir, HIDDEN_DIR_PREFIX + String(num))).
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      forEach(dir => this.removeDir(dir));
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public getExistingDownloads(): Promise<string[]> {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    const artifactFile = path.basename(this.artifactPath);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    return new Promise<string[]>((resolve, reject) => {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      fs.readdir(this.downloadsDir, (err, files) => {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        if (err) {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								          return reject(err);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        files = files.filter(file => file.endsWith(artifactFile));
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								        resolve(files);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      });
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    });
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-10 11:10:22 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  public removeUnnecessaryDownloads(existingDownloads: string[], openPrNumbers: number[]): void {
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    const toRemove = existingDownloads.filter(filePath => {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      const {pr} = getPrInfoFromDownloadPath(filePath);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      return !openPrNumbers.includes(pr);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    });
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-12 15:39:16 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    this.logger.log(`Existing downloads: ${existingDownloads.length}`);
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    this.logger.log(`Removing ${toRemove.length} download(s): ${toRemove.join(', ')}`);
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-15 15:31:39 +03:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    toRemove.forEach(filePath => shell.rm(path.join(this.downloadsDir, filePath)));
							 | 
						
					
						
							
								
									
										
										
										
											2018-05-10 13:56:07 +01:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  }
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-06 20:40:28 +02:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								}
							 |