diff --git a/dev-support/git-jira-release-audit/.gitignore b/dev-support/git-jira-release-audit/.gitignore index ddab6316242..0771f36d02e 100644 --- a/dev-support/git-jira-release-audit/.gitignore +++ b/dev-support/git-jira-release-audit/.gitignore @@ -2,3 +2,4 @@ *.log *.svg venv +new_for_*.csv diff --git a/dev-support/git-jira-release-audit/README.md b/dev-support/git-jira-release-audit/README.md index 896664e7b91..396128ad55d 100644 --- a/dev-support/git-jira-release-audit/README.md +++ b/dev-support/git-jira-release-audit/README.md @@ -43,7 +43,7 @@ $ ./venv/bin/pip install -r ./requirements.txt Successfully installed... ``` -## Usage +## Basic Usage The tool provides basic help docs. @@ -54,6 +54,7 @@ usage: git_jira_release_audit.py [-h] [--populate-from-git POPULATE_FROM_GIT] [--db-path DB_PATH] [--initialize-db INITIALIZE_DB] [--report-new-for-release-line REPORT_NEW_FOR_RELEASE_LINE] + [--report-new-for-release-branch REPORT_NEW_FOR_RELEASE_BRANCH] [--git-repo-path GIT_REPO_PATH] [--remote-name REMOTE_NAME] [--development-branch DEVELOPMENT_BRANCH] @@ -61,7 +62,9 @@ usage: git_jira_release_audit.py [-h] [--populate-from-git POPULATE_FROM_GIT] [--release-line-regexp RELEASE_LINE_REGEXP] [--parse-release-tags PARSE_RELEASE_TAGS] [--fallback-actions-path FALLBACK_ACTIONS_PATH] - [--jira-url JIRA_URL] + [--jira-url JIRA_URL] --branch-1-fix-version + BRANCH_1_FIX_VERSION --branch-2-fix-version + BRANCH_2_FIX_VERSION optional arguments: -h, --help show this help message and exit @@ -69,90 +72,169 @@ optional arguments: Building the audit database: --populate-from-git POPULATE_FROM_GIT When true, populate the audit database from the Git - repository. + repository. (default: True) --populate-from-jira POPULATE_FROM_JIRA When true, populate the audit database from Jira. + (default: True) --db-path DB_PATH Path to the database file, or leave unspecified for a - transient db. + transient db. (default: audit.db) --initialize-db INITIALIZE_DB When true, initialize the database tables. This is destructive to the contents of an existing database. + (default: False) Generating reports: --report-new-for-release-line REPORT_NEW_FOR_RELEASE_LINE Builds a report of the Jira issues that are new on the target release line, not present on any of the associated release branches. (i.e., on branch-2 but - not branch-{2.0,2.1,...}) + not branch-{2.0,2.1,...}) (default: None) + --report-new-for-release-branch REPORT_NEW_FOR_RELEASE_BRANCH + Builds a report of the Jira issues that are new on the + target release branch, not present on any of the + previous release branches. (i.e., on branch-2.3 but + not branch-{2.0,2.1,...}) (default: None) Interactions with the Git repo: --git-repo-path GIT_REPO_PATH Path to the git repo, or leave unspecified to infer - from the current file's path. + from the current file's path. (default: + ./git_jira_release_audit.py) --remote-name REMOTE_NAME The name of the git remote to use when identifying - branches. Default: 'origin' + branches. Default: 'origin' (default: origin) --development-branch DEVELOPMENT_BRANCH The name of the branch from which all release lines - originate. Default: 'master' + originate. Default: 'master' (default: master) --development-branch-fix-version DEVELOPMENT_BRANCH_FIX_VERSION The Jira fixVersion used to indicate an issue is - committed to the development branch. Default: '3.0.0' + committed to the development branch. (default: 3.0.0) --release-line-regexp RELEASE_LINE_REGEXP - A regexp used to identify release lines. + A regexp used to identify release lines. (default: + branch-\d+$) --parse-release-tags PARSE_RELEASE_TAGS When true, look for release tags and annotate commits according to their release version. An Expensive - calculation, disabled by default. + calculation, disabled by default. (default: False) --fallback-actions-path FALLBACK_ACTIONS_PATH Path to a file containing _DB.Actions applicable to - specific git shas. + specific git shas. (default: fallback_actions.csv) --branch-1-fix-version BRANCH_1_FIX_VERSION The Jira fixVersion used to indicate an issue is committed to the specified release line branch + (default: None) --branch-2-fix-version BRANCH_2_FIX_VERSION The Jira fixVersion used to indicate an issue is committed to the specified release line branch + (default: None) Interactions with Jira: - --jira-url JIRA_URL A URL locating the target JIRA instance. + --jira-url JIRA_URL A URL locating the target JIRA instance. (default: + https://issues.apache.org/jira) ``` +### Build a Database + +This invocation will build a "simple" database, correlating commits to +branches. It omits gathering the detailed release tag data, so it runs pretty +quickly. + Example Run: ```shell script $ ./venv/bin/python3 ./git_jira_release_audit.py \ --db-path=audit.db \ - --remote-name=apache-rw \ --development-branch-fix-version=3.0.0 \ - --branch-1-fix-version=1.5.0 \ - --branch-2-fix-version=2.3.0 -INFO:root:apache-rw/branch-1 has 4046 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. -INFO:root:apache-rw/branch-1.0 has 1433 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. -INFO:root:apache-rw/branch-1.1 has 2111 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. -INFO:root:apache-rw/branch-1.2 has 2738 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. -INFO:root:apache-rw/branch-1.3 has 3287 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. -INFO:root:apache-rw/branch-1.4 has 3912 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. -INFO:root:apache-rw/branch-2 has 3080 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. -INFO:root:apache-rw/branch-2.0 has 2194 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. -INFO:root:apache-rw/branch-2.1 has 2705 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. -INFO:root:apache-rw/branch-2.2 has 2927 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. -INFO:root:retrieving 5653 jira_ids from the issue tracker + --branch-1-fix-version=1.7.0 \ + --branch-2-fix-version=2.4.0 +INFO:git_jira_release_audit.py:origin/branch-1.0 has 1433 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. +INFO:git_jira_release_audit.py:origin/branch-1.1 has 2111 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. +INFO:git_jira_release_audit.py:origin/branch-1.2 has 2738 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. +INFO:git_jira_release_audit.py:origin/branch-1.3 has 3296 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. +INFO:git_jira_release_audit.py:origin/branch-1.4 has 3926 commits since its origin at 0167558eb31ff48308d592ef70b6d005ba6d21fb. +INFO:git_jira_release_audit.py:origin/branch-2 has 3325 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. +INFO:git_jira_release_audit.py:origin/branch-2.0 has 2198 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. +INFO:git_jira_release_audit.py:origin/branch-2.1 has 2749 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. +INFO:git_jira_release_audit.py:origin/branch-2.2 has 2991 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. +INFO:git_jira_release_audit.py:origin/branch-2.3 has 3312 commits since its origin at 0d0c330401ade938bf934aafd79ec23705edcc60. +INFO:git_jira_release_audit.py:retrieving 5850 jira_ids from the issue tracker -apache-rw/branch-1 100%|██████████████████████████████████████████████████████| 4046/4046 [08:23<00:00, 8.04 commit/s] -apache-rw/branch-1.0 100%|████████████████████████████████████████████████████| 1433/1433 [03:49<00:00, 6.26 commit/s] -apache-rw/branch-1.1 100%|████████████████████████████████████████████████████| 2111/2111 [05:16<00:00, 6.68 commit/s] -apache-rw/branch-1.2 100%|████████████████████████████████████████████████████| 2738/2738 [06:26<00:00, 7.10 commit/s] -apache-rw/branch-1.3 100%|████████████████████████████████████████████████████| 3287/3287 [07:21<00:00, 7.46 commit/s] -apache-rw/branch-1.4 100%|████████████████████████████████████████████████████| 3912/3912 [08:08<00:00, 8.02 commit/s] -apache-rw/branch-2 100%|█████████████████████████████████████████████████████| 3080/3080 [03:29<00:00, 14.74 commit/s] -apache-rw/branch-2.0 100%|████████████████████████████████████████████████████| 2194/2194 [04:56<00:00, 7.42 commit/s] -apache-rw/branch-2.1 100%|███████████████████████████████████████████████████| 2705/2705 [03:17<00:00, 13.75 commit/s] -apache-rw/branch-2.2 100%|███████████████████████████████████████████████████| 2927/2927 [03:28<00:00, 14.09 commit/s] -fetch from Jira 100%|█████████████████████████████████████████████████████████| 5653/5653 [00:58<00:00, 98.29 issue/s] +origin/branch-1 100%|████████████████████████████████████| 4084/4084 [00:00<00:00, 9805.33 commit/s] +origin/branch-1.0 100%|█████████████████████████████████| 1433/1433 [00:00<00:00, 10479.89 commit/s] +origin/branch-1.1 100%|█████████████████████████████████| 2111/2111 [00:00<00:00, 10280.60 commit/s] +origin/branch-1.2 100%|██████████████████████████████████| 2738/2738 [00:00<00:00, 8833.51 commit/s] +origin/branch-1.3 100%|██████████████████████████████████| 3296/3296 [00:00<00:00, 9746.93 commit/s] +origin/branch-1.4 100%|██████████████████████████████████| 3926/3926 [00:00<00:00, 9750.96 commit/s] +origin/branch-2 100%|████████████████████████████████████| 3325/3325 [00:00<00:00, 9688.14 commit/s] +origin/branch-2.0 100%|██████████████████████████████████| 2198/2198 [00:00<00:00, 8804.18 commit/s] +origin/branch-2.1 100%|██████████████████████████████████| 2749/2749 [00:00<00:00, 9328.67 commit/s] +origin/branch-2.2 100%|██████████████████████████████████| 2991/2991 [00:00<00:00, 9215.56 commit/s] +origin/branch-2.3 100%|██████████████████████████████████| 3312/3312 [00:00<00:00, 9063.19 commit/s] +fetch from Jira 100%|████████████████████████████████████████| 5850/5850 [10:40<00:00, 9.14 issue/s] ``` -With a populated database, query with sqlite: +Optionally, the database can be build to include release tags, by specifying +`--parse-release-tags=true`. This is more time-consuming, but is necessary for +auditing discrepancies between git and Jira. Running the same command but +including this flag looks like this: + +```shell script +origin/branch-1 100%|███████████████████████████████████████| 4084/4084 [08:58<00:00, 7.59 commit/s] +origin/branch-1.0 100%|█████████████████████████████████████| 1433/1433 [03:54<00:00, 6.13 commit/s] +origin/branch-1.1 100%|█████████████████████████████████████| 2111/2111 [41:26<00:00, 0.85 commit/s] +origin/branch-1.2 100%|█████████████████████████████████████| 2738/2738 [07:10<00:00, 6.37 commit/s] +origin/branch-1.3 100%|██████████████████████████████████| 3296/3296 [2h 33:13<00:00, 0.36 commit/s] +origin/branch-1.4 100%|██████████████████████████████████| 3926/3926 [7h 22:41<00:00, 0.15 commit/s] +origin/branch-2 100%|████████████████████████████████████| 3325/3325 [2h 05:43<00:00, 0.44 commit/s] +origin/branch-2.0 100%|█████████████████████████████████████| 2198/2198 [52:18<00:00, 0.70 commit/s] +origin/branch-2.1 100%|█████████████████████████████████████| 2749/2749 [17:09<00:00, 2.67 commit/s] +origin/branch-2.2 100%|█████████████████████████████████████| 2991/2991 [52:15<00:00, 0.95 commit/s] +origin/branch-2.3 100%|████████████████████████████████████| 3312/3312 [05:08<00:00, 10.74 commit/s] +fetch from Jira 100%|████████████████████████████████████████| 5850/5850 [10:46<00:00, 9.06 issue/s] +``` + +### Run a Report + +With a database populated with branch information, the build-in reports can be +run. + +`--report-new-for-release-line` +> Builds a report of the Jira issues that are new on the target release line, +not present on any of the associated release branches. (i.e., on branch-2 but +not branch-{2.0,2.1,...}) + +`--report-new-for-release-branch` +> Builds a report of the Jira issues that are new on the target release branch, +not present on any of the previous release branches. (i.e., on branch-2.3 but +not branch-{2.0,2.1,...}) + +Either way, the output is a csv file containing a summary of each JIRA id found +matching the report criteria. + +Example Run: + +```shell script +$ ./venv/bin/python3.7 ./git_jira_release_audit.py \ + --populate-from-git=false \ + --populate-from-jira=false \ + --branch-1-fix-version=1.7.0 \ + --branch-2-fix-version=2.4.0 \ + --report-new-for-release-branch=origin/branch-2.3 +INFO:git_jira_release_audit.py:retrieving 292 jira_ids from the issue tracker +INFO:git_jira_release_audit.py:generated report at new_for_origin-branch-2.3.csv + +fetch from Jira 100%|████████████████████████████████████████| 292/292 [00:03<00:00, 114.01 issue/s] +$ head -n5 new_for_origin-branch-2.3.csv +key,issue_type,priority,summary,resolution,components +HBASE-21070,Bug,Critical,SnapshotFileCache won't update for snapshots stored in S3,Fixed,['snapshots'] +HBASE-21773,Bug,Critical,rowcounter utility should respond to pleas for help,Fixed,['tooling'] +HBASE-21505,Bug,Major,Several inconsistencies on information reported for Replication Sources by hbase shell status 'replication' command.,Fixed,['Replication'] +HBASE-22057,Bug,Major,Impose upper-bound on size of ZK ops sent in a single multi(),Fixed,[] +``` + +### Explore the Database + +With a populated database, query it with sqlite: ```shell script $ sqlite3 audit.db @@ -160,56 +242,71 @@ SQLite version 3.24.0 2018-06-04 14:10:15 Enter ".help" for usage hints. sqlite> -- count the number of distinct commits on a release branch sqlite> select count(distinct jira_id), branch from git_commits group by branch; -3406|apache-rw/branch-1 -1189|apache-rw/branch-1.0 -1728|apache-rw/branch-1.1 -2289|apache-rw/branch-1.2 -2779|apache-rw/branch-1.3 -3277|apache-rw/branch-1.4 -2666|apache-rw/branch-2 -1809|apache-rw/branch-2.0 -2289|apache-rw/branch-2.1 -2511|apache-rw/branch-2.2 - -sqlite> -- count the number of issues that will be in 2.3.0 that have not been released on any earlier -sqlite> -- version. -sqlite> select count(1) from ( - select distinct jira_id from git_commits where branch = 'apache-rw/branch-2' except - select distinct jira_id from git_commits where branch in - ('apache-rw/branch-2.0', 'apache-rw/branch-2.1', 'apache-rw/branch-2.2')); -169 +3437|origin/branch-1 +1189|origin/branch-1.0 +1728|origin/branch-1.1 +2289|origin/branch-1.2 +2788|origin/branch-1.3 +3289|origin/branch-1.4 +2846|origin/branch-2 +1813|origin/branch-2.0 +2327|origin/branch-2.1 +2566|origin/branch-2.2 +2839|origin/branch-2.3 sqlite> -- find the issues for which the git commit record and JIRA fixVersion disagree +sqlite> -- this query requires the database be built with --parse-release-tags sqlite> select g.jira_id, g.git_tag, j.fix_version from git_commits g inner join jira_versions j on g.jira_id = j.jira_id - and g.branch = 'apache-rw/branch-2.2' + and g.branch = 'origin/branch-2.2' and g.git_tag is not null and j.fix_version like '2.2.%' and g.git_tag != j.fix_version; HBASE-22941|2.2.2|2.2.1 -sqlite> -- show jira non-1.x fixVersions for all issues on branch-2 but not on any -sqlite> -- branch-2.x release branch; i.e., issues that are missing a fixVersion or -sqlite> -- are marked for a release other than (3.0.0, 2.3.0) -sqlite> select g.jira_id, j.fix_version -from ( - select distinct jira_id from git_commits where branch = 'apache-rw/branch-2' except - select distinct jira_id from git_commits where branch in - (select distinct branch from git_commits where branch like 'apache-rw/branch-2.%')) g -left join jira_versions j - on g.jira_id = j.jira_id - and j.fix_version not like '1.%' -where ( - j.fix_version is null - OR j.fix_version not in ('3.0.0', '2.3.0')) -order by g.jira_id desc; -HBASE-23683|2.2.4 +sqlite> -- show jira fixVersions for all issues on branch-2.3 but not on any earlier +sqlite> -- branch; i.e., issues that are missing a fixVersion or are marked for +sqlite> -- a release other than the expected (3.0.0, 2.3.0). +sqlite> -- this query requires the database be built with --parse-release-tags +sqlite> select jira_id, fix_version + FROM jira_versions + WHERE jira_id in ( + SELECT distinct jira_id + FROM git_commits + WHERE branch = 'origin/branch-2.3' + EXCEPT SELECT distinct jira_id + FROM git_commits + WHERE branch IN ( + SELECT distinct branch + FROM git_commits + WHERE branch != 'origin/branch-2.3')) + AND fix_version NOT IN ('3.0.0', '2.3.0') + ORDER BY jira_id; +HBASE-22321|1.5.0 +HBASE-22360|2.2.0 +HBASE-22405|2.2.0 +HBASE-22555|2.4.0 HBASE-23032|connector-1.0.1 HBASE-23032|hbase-filesystem-1.0.0-alpha2 -HBASE-22405|2.2.0 -HBASE-22360|2.2.0 -HBASE-22321| -HBASE-22283|2.2.0 +HBASE-23604|HBASE-18095 +HBASE-23633|2.4.0 +HBASE-23647|HBASE-18095 +HBASE-23648|HBASE-18095 +HBASE-23731|HBASE-18095 +HBASE-23741|2.4.0 +HBASE-23752|HBASE-18095 +HBASE-23804|HBASE-18095 +HBASE-23851|master +HBASE-23936|2.4.0 +HBASE-23937|2.4.0 +HBASE-23977|2.4.0 +HBASE-24002|2.4.0 +HBASE-24033|2.4.0 +HBASE-24037|2.4.0 +HBASE-24073|master +HBASE-24075|2.4.0 +HBASE-24080|2.4.0 +HBASE-24080|master ``` diff --git a/dev-support/git-jira-release-audit/fallback_actions.csv b/dev-support/git-jira-release-audit/fallback_actions.csv index 0604eeb21e4..eb6c97c1c56 100644 --- a/dev-support/git-jira-release-audit/fallback_actions.csv +++ b/dev-support/git-jira-release-audit/fallback_actions.csv @@ -27,8 +27,10 @@ hexsha,action,jira_id 05f8e94191ef6a63baadf56d6114d7d0317796f2,SKIP, 0791b878422eadf00b55076338f09bf059f39f0c,SKIP, 07f9f3d38cf4d0d01044ab28d90a50a1a009f6b8,SKIP, +0bff1305134b9c3a0bcad21900f5af68a8aedb4a,SKIP, 10f00547627076d79d77cf58dd2deaece2287084,ADD,HBASE-22330 10f3b77748a02a2c11635c33964929c0474e890d,SKIP, +1196e42362312080d3c523c107b5e8fefef9e57e,SKIP, 1404d5a97331ecc63db53971f5cb7329cb40ce67,ADD,HBASE-15203 14a869828fe481697d29b2d6e4135e8026039a38,SKIP, 1546613e76b1013a08ebc179c2c22bfeb44f3a4a,SKIP, @@ -42,6 +44,7 @@ hexsha,action,jira_id 1b3557649c9ee682c7f135ca52a0e3cd10cb9219,SKIP, 1c46250bef9ef9be9c255d61bda69ff7792ed551,SKIP, 1cb7d0e82ad64f37fbd6de950b74081b0d5eddf3,SKIP, +1d988afc9d2065a51fe74d0553f0943ef540dfaa,SKIP, 1eaef185327171b3dd3edb303e08cfe85186e745,SKIP, 1eb8ac6fe9dd0c15cdb52f66ced4136316c06465,SKIP, 2068804d7510e8c1f822b5db3cd4585455f6e7e7,SKIP, @@ -52,6 +55,7 @@ hexsha,action,jira_id 259d12f7397679c6b0d0a4788e5a37f65fd49f20,SKIP, 267bce0590c39570ddb935921e34bda35e3aa44c,SKIP, 278828333c44493ccbaa7db26a788b2756632034,SKIP, +27c1f2f978142b7bb4135e53c3ae067c5608c9fb,SKIP, 288794d68ba5bd4d1fd8d5c315cee972019dcb3d,ADD,HBASE-22330 28f07451a5dddf0ab3988b32b8672654fdbc5b58,SKIP, 2ba542d74c2d9e78332c8c94289d1295752d8072,SKIP, @@ -97,7 +101,9 @@ hexsha,action,jira_id 4eb84651a2b6d02d2074143308cef5d0f4b856a3,SKIP, 4f5b22bc19cb8d24ced5d42ebd9794cfd83bae85,SKIP, 54337870eda5649ab7bb81ed01c9dd25d59204f2,SKIP, +558ee079fd04dfab8e61eca10ee98ab5bac89dfa,SKIP, 58ab201be341f02829286f036a7401d0806eb999,SKIP, +58b63e04c4af99a5730efb0a7e553be4d950e6a5,SKIP, 5a16c15d7f51087a50511a2e0730f547c97a033f,SKIP, 5b5ff1d8b2cc43f78acaf9bc960be382dc6c34f7,SKIP, 5fa15dd7488433ea610ff5e92161409d20565690,SKIP, @@ -170,6 +176,7 @@ b3d55441b8174c704ada4585603f6bcfca298843,SKIP, b65231d04dbc565a578ce928e809aa51f5439857,SKIP, b6549007b313e8f3aa993d5c1ebd29c84ccb7b7b,SKIP, b6d4fc955fe0fc41f5225f1cc2e3e4b92029251c,SKIP, +b9c676cdc048c52f927cfa906fd18ff412e4ca20,SKIP, b9f5c6b065ebd572193c1fdc9d38557320b42fe6,SKIP, bcadcef21048e4764f7ae8dec3ce52884f20c02c,SKIP, bcdc56ac76e4a26e53faa8301a441e94ee8614d7,SKIP, @@ -178,9 +185,11 @@ bd4e14db07ea32a45c3ef734e06d195a405da67c,SKIP, bd4eba2b53b7af738fd9584511d737c4393d0855,SKIP, bef0616ef33306afca3060b96c2cba5f9762035d,SKIP, c100fb835a54be6002fe9704349e726f27b15b7a,SKIP, +c5e0a1397b3c6a14612e4c5b66f995c02de4310b,SKIP, c71da858ada94e1b93065f0b7caf3558942bc4da,SKIP, c89cfd3406823cf05fa83464c5ddee16bf0d473f,ADD,HBASE-17248 c89cfd3406823cf05fa83464c5ddee16bf0d473f,ADD,HBASE-17248 +c8c2a875056f27c9af81293a504d75634cbc1fa5,SKIP, c97905a962b88a0c68ca8a51c2e507daec81ca6d,SKIP, c9f506a2973e0acbd0d2df7b9353c9291f6c94a8,SKIP, cbb2c7e00d0c0b3f641250d981b9c87286d31058,ADD,HBASE-23069 @@ -209,6 +218,7 @@ e40fcee6b54712b76d702af6937c3320c60df2b9,SKIP, e501fe1a296be8fec0890e7e15414683aa3d933b,SKIP, e5349d589c000e395e12340e003aa9e2153afea6,SKIP, e5fb8214b2bfd6396539a4e8b6cf5f3cc5e9c06f,REVERT,HBASE-21874 +e869a20123afe326e198d35d110f5c0360ea244f,SKIP, e8e45ef8f2fb91a870399636b492d5cee58a4c39,SKIP, e92a147e1961366e36a39577816994566e1e21c5,SKIP, eacf3cb29641af1a68978d9bd7654f643a3aa3a1,SKIP, diff --git a/dev-support/git-jira-release-audit/git_jira_release_audit.py b/dev-support/git-jira-release-audit/git_jira_release_audit.py index 7eac6d33d47..db2788d081d 100644 --- a/dev-support/git-jira-release-audit/git_jira_release_audit.py +++ b/dev-support/git-jira-release-audit/git_jira_release_audit.py @@ -30,12 +30,16 @@ import pathlib import re import sqlite3 import time +import os import enlighten import git import jira +LOG = logging.getLogger(os.path.basename(__file__)) + + class _DB: """Manages an instance of Sqlite on behalf of the application. @@ -46,6 +50,9 @@ class _DB: Attributes: conn (:obj:`sqlite3.db2api.Connection`): The underlying connection object. """ + + SQL_LOG = LOG.getChild("sql") + class Action(enum.Enum): """Describes an action to be taken against the database.""" ADD = 'ADD' @@ -54,6 +61,7 @@ class _DB: def __init__(self, db_path, initialize_db, **_kwargs): self._conn = sqlite3.connect(db_path) + self._conn.set_trace_callback(_DB.log_query) if initialize_db: for table in 'git_commits', 'jira_versions': @@ -81,6 +89,10 @@ class _DB: def __exit__(self, exc_type, exc_val, exc_tb): self._conn.close() + @staticmethod + def log_query(query): + _DB.SQL_LOG.debug(re.sub(r'\s+', ' ', query).strip()) + @property def conn(self): """:obj:`sqlite3.db2api.Connection`: Underlying database handle.""" @@ -324,7 +336,7 @@ class _RepoReader: def _resolve_ambiguity(self, commit): if commit.hexsha not in self._fallback_actions: - logging.warning('Unable to resolve action for %s: %s', commit.hexsha, commit.summary) + LOG.warning('Unable to resolve action for %s: %s', commit.hexsha, commit.summary) return _DB.Action.SKIP, None action, jira_id = self._fallback_actions[commit.hexsha] if not jira_id: @@ -354,7 +366,7 @@ class _RepoReader: global MANAGER commits = list(self._repo.iter_commits( "%s...%s" % (origin_commit.hexsha, release_branch), reverse=True)) - logging.info("%s has %d commits since its origin at %s.", release_branch, len(commits), + LOG.info("%s has %d commits since its origin at %s.", release_branch, len(commits), origin_commit) counter = MANAGER.counter(total=len(commits), desc=release_branch, unit='commit') commits_since_release = list() @@ -404,7 +416,7 @@ class _JiraReader: database.""" global MANAGER jira_ids = self._db.unique_jira_ids_from_git() - logging.info("retrieving %s jira_ids from the issue tracker", len(jira_ids)) + LOG.info("retrieving %s jira_ids from the issue tracker", len(jira_ids)) counter = MANAGER.counter(total=len(jira_ids), desc='fetch from Jira', unit='issue') chunk_size = 50 chunks = [jira_ids[i:i + chunk_size] for i in range(0, len(jira_ids), chunk_size)] @@ -429,7 +441,7 @@ class _JiraReader: def fetch_issues(self, jira_ids): """Retrieve the specified jira Ids.""" global MANAGER - logging.info("retrieving %s jira_ids from the issue tracker", len(jira_ids)) + LOG.info("retrieving %s jira_ids from the issue tracker", len(jira_ids)) counter = MANAGER.counter(total=len(jira_ids), desc='fetch from Jira', unit='issue') chunk_size = 50 chunks = [jira_ids[i:i + chunk_size] for i in range(0, len(jira_ids), chunk_size)] @@ -501,17 +513,17 @@ class Auditor: writer.writeheader() for issue in issues: writer.writerow(issue) - logging.info('generated report at %s', filename) + LOG.info('generated report at %s', filename) def report_new_for_release_line(self, release_line): """Builds a report of the Jira issues that are new on the target release line, not present on any of the associated release branches. (i.e., on branch-2 but not branch-{2.0,2.1,...})""" matches = [x for x in self._repo_reader.release_line_refs - if x.name == release_line or x.name.endswith('/%s' % release_line)] + if x.name == release_line or x.remote_head == release_line] release_line_ref = next(iter(matches), None) if not release_line_ref: - logging.error('release line %s not found. available options are %s.', + LOG.error('release line %s not found. available options are %s.', release_line, [x.name for x in self._repo_reader.release_line_refs]) return cursor = self._db.conn.execute(""" @@ -525,6 +537,31 @@ class Auditor: filename = 'new_for_%s.csv' % release_line.replace('/', '-') Auditor._write_report(filename, issues) + def report_new_for_release_branch(self, release_branch): + """Builds a report of the Jira issues that are new on the target release branch, not present + on any of the previous release branches. (i.e., on branch-2.3 but not + branch-{2.0,2.1,...})""" + matches = [x for x in self._repo_reader.release_branch_refs + if x.name == release_branch or x.remote_head == release_branch] + release_branch_ref = next(iter(matches), None) + if not release_branch_ref: + LOG.error('release branch %s not found. available options are %s.', + release_branch, [x.name for x in self._repo_reader.release_branch_refs]) + return + previous_branches = [x.name for x in self._repo_reader.release_branch_refs + if x.remote_head != release_branch_ref.remote_head] + query = ( + "SELECT distinct jira_id FROM git_commits" + " WHERE branch = ?" + " EXCEPT SELECT distinct jira_id FROM git_commits" + f" WHERE branch IN ({','.join('?' for _ in previous_branches)})" + ) + cursor = self._db.conn.execute(query, tuple([release_branch_ref.name] + previous_branches)) + jira_ids = [x[0] for x in cursor.fetchall()] + issues = self._jira_reader.fetch_issues(jira_ids) + filename = 'new_for_%s.csv' % release_branch.replace('/', '-') + Auditor._write_report(filename, issues) + @staticmethod def _str_to_bool(val): if not val: @@ -548,7 +585,7 @@ class Auditor: building_group.add_argument( '--db-path', help='Path to the database file, or leave unspecified for a transient db.', - default=':memory:') + default='audit.db') building_group.add_argument( '--initialize-db', help='When true, initialize the database tables. This is destructive to the contents' @@ -561,6 +598,11 @@ class Auditor: help=Auditor.report_new_for_release_line.__doc__, type=str, default=None) + report_group.add_argument( + '--report-new-for-release-branch', + help=Auditor.report_new_for_release_branch.__doc__, + type=str, + default=None) git_repo_group = parser.add_argument_group('Interactions with the Git repo') git_repo_group.add_argument( '--git-repo-path', @@ -580,7 +622,7 @@ class Auditor: git_repo_group.add_argument( '--development-branch-fix-version', help='The Jira fixVersion used to indicate an issue is committed to the development' - + ' branch. Default: \'3.0.0\'', + + ' branch.', default='3.0.0') git_repo_group.add_argument( '--release-line-regexp', @@ -612,7 +654,10 @@ class Auditor: help='The Jira fixVersion used to indicate an issue is committed to the specified ' + 'release line branch', required=True) - return argparse.ArgumentParser(parents=[parent_parser]) + return argparse.ArgumentParser( + parents=[parent_parser], + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) MANAGER = None @@ -621,11 +666,11 @@ MANAGER = None def main(): global MANAGER + logging.basicConfig(level=logging.INFO) first_pass_parser, git_repo_group = Auditor._build_first_pass_parser() first_pass_args, extras = first_pass_parser.parse_known_args() first_pass_args_dict = vars(first_pass_args) with _DB(**first_pass_args_dict) as db: - logging.basicConfig(level=logging.INFO) repo_reader = _RepoReader(db, **first_pass_args_dict) jira_reader = _JiraReader(db, **first_pass_args_dict) second_pass_parser = Auditor._build_second_pass_parser( @@ -641,6 +686,9 @@ def main(): if second_pass_args.report_new_for_release_line: release_line = second_pass_args.report_new_for_release_line auditor.report_new_for_release_line(release_line) + if second_pass_args.report_new_for_release_branch: + release_branch = second_pass_args.report_new_for_release_branch + auditor.report_new_for_release_branch(release_branch) if __name__ == '__main__':