From 5206952c23031b8f3e3955c9329cdc2af9dad815 Mon Sep 17 00:00:00 2001
From: Jack McCluskey <34928439+jrmccluskey@users.noreply.github.com>
Date: Tue, 6 Jun 2023 13:22:09 -0400
Subject: [PATCH] Remove Beam Dependency Report code (#26954)
* Remove Beam Dependency Report code
* Remove reference in README
* Remove other references to the dependency job
* More references
---
.test-infra/jenkins/README.md | 1 -
.../jenkins/dependency_check/__init__.py | 16 -
.../dependency_check/bigquery_client_utils.py | 149 --------
.../dependency_check_report_generator.py | 350 ------------------
.../dependency_check_report_generator_test.py | 134 -------
.../dependency_check/generate_report.sh | 64 ----
.../report_generator_config.py | 81 ----
.../dependency_check/version_comparer.py | 51 ---
.../dependency_check/version_comparer_test.py | 54 ---
.../jenkins/job_Dependency_Check.groovy | 73 ----
build.gradle.kts | 10 -
ownership/JAVA_DEPENDENCY_OWNERS.yaml | 5 -
sdks/python/build.gradle | 5 -
sdks/python/scripts/run_dependency_check.sh | 34 --
.../test-suites/tox/pycommon/build.gradle | 4 +-
sdks/python/tox.ini | 10 -
16 files changed, 1 insertion(+), 1040 deletions(-)
delete mode 100644 .test-infra/jenkins/dependency_check/__init__.py
delete mode 100644 .test-infra/jenkins/dependency_check/bigquery_client_utils.py
delete mode 100644 .test-infra/jenkins/dependency_check/dependency_check_report_generator.py
delete mode 100644 .test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py
delete mode 100755 .test-infra/jenkins/dependency_check/generate_report.sh
delete mode 100644 .test-infra/jenkins/dependency_check/report_generator_config.py
delete mode 100644 .test-infra/jenkins/dependency_check/version_comparer.py
delete mode 100644 .test-infra/jenkins/dependency_check/version_comparer_test.py
delete mode 100644 .test-infra/jenkins/job_Dependency_Check.groovy
delete mode 100755 sdks/python/scripts/run_dependency_check.sh
diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md
index bdd3f2577ae1c..5f9b55d4e2366 100644
--- a/.test-infra/jenkins/README.md
+++ b/.test-infra/jenkins/README.md
@@ -285,7 +285,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/)
| Name | Link | PR Trigger Phrase | Cron Status |
|------|------|-------------------|-------------|
-| beam_Dependency_Check | [cron](https://ci-beam.apache.org/job/beam_Dependency_Check/) | `Run Dependency Check` | [![Build Status](https://ci-beam.apache.org/job/beam_Dependency_Check/badge/icon)](https://ci-beam.apache.org/job/beam_Dependency_Check) |
| beam_Metrics_Report | [cron](https://ci-beam.apache.org/job/beam_Metrics_Report/) | `Run Metrics Report` | [![Build Status](https://ci-beam.apache.org/job/beam_Metrics_Report/badge/icon)](https://ci-beam.apache.org/job/beam_Metrics_Report) |
| beam_Release_NightlySnapshot | [cron](https://ci-beam.apache.org/job/beam_Release_NightlySnapshot/) | `Run Gradle Publish` | [![Build Status](https://ci-beam.apache.org/job/beam_Release_NightlySnapshot/badge/icon)](https://ci-beam.apache.org/job/beam_Release_NightlySnapshot) |
| beam_Release_Python_NightlySnapshot | [cron](https://ci-beam.apache.org/job/beam_Release_Python_NightlySnapshot/) | `Run Python Publish` | [![Build Status](https://ci-beam.apache.org/job/beam_Release_Python_NightlySnapshot/badge/icon)](https://ci-beam.apache.org/job/beam_Release_Python_NightlySnapshot) |
diff --git a/.test-infra/jenkins/dependency_check/__init__.py b/.test-infra/jenkins/dependency_check/__init__.py
deleted file mode 100644
index cce3acad34a49..0000000000000
--- a/.test-infra/jenkins/dependency_check/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
diff --git a/.test-infra/jenkins/dependency_check/bigquery_client_utils.py b/.test-infra/jenkins/dependency_check/bigquery_client_utils.py
deleted file mode 100644
index f7cd0fe24b9e4..0000000000000
--- a/.test-infra/jenkins/dependency_check/bigquery_client_utils.py
+++ /dev/null
@@ -1,149 +0,0 @@
-from __future__ import print_function
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import datetime
-import logging
-from google.cloud import bigquery
-
-logging.getLogger().setLevel(logging.INFO)
-
-class BigQueryClientUtils:
-
- def __init__(self, project_id, dataset_id, table_id):
- self.project_id = project_id
- self.dataset_id = dataset_id
- self.table_id = table_id
- self.bigquery_client = bigquery.Client(project_id)
- self.table_ref = self.bigquery_client.dataset(dataset_id).table(table_id)
- self.table = self.bigquery_client.get_table(self.table_ref)
-
-
- def query_dep_info_by_version(self, dep, version):
- """
- Query for dependency information of a specific version
- Args:
- dep, version
- Return:
- release_date, is_currently_used
- """
- query = """SELECT release_date, is_currently_used
- FROM `{0}.{1}.{2}`
- WHERE package_name=\'{3}\' AND version=\'{4}\'""".format(self.project_id,
- self.dataset_id,
- self.table_id,
- dep.strip(),
- version.strip())
-
- query_job = self.bigquery_client.query(query)
- rows = list(query_job)
- if len(rows) == 0:
- logging.info("Did not find record of dependency {0} with version {1}.".format(dep, version))
- return None, False
- assert len(rows) == 1
- logging.info("""Found record of dependency {0} with version {1}:
- release date: {2}; is_currently_used: {3}.""".format(dep, version, rows[0]['release_date'], rows[0]['is_currently_used']))
- return rows[0]['release_date'], rows[0]['is_currently_used']
-
-
- def query_currently_used_dep_info_in_db(self, dep):
- """
- Query for the info of the currently used version of a specific dependency
- Args:
- dep
- Return:
- version, release_date
- """
- query = """SELECT version, release_date
- FROM `{0}.{1}.{2}`
- WHERE package_name=\'{3}\' AND is_currently_used=True""".format(self.project_id,
- self.dataset_id,
- self.table_id,
- dep.strip())
-
- query_job = self.bigquery_client.query(query)
- rows = list(query_job)
- if len(rows) == 0:
- return None, None
- assert len(rows) == 1
- return rows[0]['version'], rows[0]['release_date']
-
-
- def insert_dep_to_table(self, dep, version, release_date, is_currently_used=False):
- """
- Add a dependency with version and release date into bigquery table
- Args:
- dep, version, is_currently_used (default False)
- """
- query = """INSERT
- `{0}.{1}.{2}` (package_name, version, release_date, is_currently_used)
- VALUES (\'{3}\', \'{4}\', \'{5}\', {6})""".format(self.project_id,
- self.dataset_id,
- self.table_id,
- dep.strip(),
- version.strip(),
- release_date,
- is_currently_used)
- logging.info("Inserting dep to table: \n {0}".format(query))
- try:
- query_job = self.bigquery_client.query(query)
- if not query_job.done():
- print(query_job.result())
- except:
- raise
-
-
- def delete_dep_from_table(self, dep, version):
- """
- Remove a dependency record from the table.
- Args:
- dep, version
- """
- query = """DELETE
- FROM `{0}.{1}.{2}`
- WHERE package_name=\'{3}\' AND version=\'{4}\'""".format(self.project_id,
- self.dataset_id,
- self.table_id,
- dep.strip(),
- version.strip())
- logging.info("Deleting dep from table: \n {0}".format(query))
- try:
- query_job = self.bigquery_client.query(query)
- if not query_job.done():
- print(query_job.result())
- except:
- raise
-
-
- def clean_stale_records_from_table(self):
- """
- Remove stale records from the table. A record is stale if it is not currently used and the release date is behind 3
- years.
- """
- query = """DELETE
- FROM `{0}.{1}.{2}`
- WHERE release_date < '{3}'""".format(self.project_id,
- self.dataset_id,
- self.table_id,
- datetime.datetime.today().date() - datetime.timedelta(3*365))
- logging.info("Clean Up Starts")
- try:
- query_job = self.bigquery_client.query(query)
- if not query_job.done():
- logging.error(query_job.result())
- except:
- raise
diff --git a/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py b/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py
deleted file mode 100644
index a7f6035e4e2bc..0000000000000
--- a/.test-infra/jenkins/dependency_check/dependency_check_report_generator.py
+++ /dev/null
@@ -1,350 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import dependency_check.version_comparer as version_comparer
-import logging
-import os.path
-import re
-import requests
-import sys
-import traceback
-
-from datetime import datetime
-from dependency_check.bigquery_client_utils import BigQueryClientUtils
-from dependency_check.report_generator_config import ReportGeneratorConfig
-from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util.retry import Retry
-
-
-logging.getLogger().setLevel(logging.INFO)
-
-class InvalidFormatError(Exception):
- def __init__(self, message):
- super(InvalidFormatError, self).__init__(message)
-
-
-def extract_results(file_path):
- """
- Extract the Java/Python dependency reports and return a collection of out-of-date dependencies.
- Args:
- file_path: the path of the raw reports
- Return:
- outdated_deps: a collection of dependencies that has updates
- """
- outdated_deps = []
- try:
- with open(file_path) as raw_report:
- see_oudated_deps = False
- for line in raw_report:
- if see_oudated_deps:
- outdated_deps.append(line)
- if line.startswith('The following dependencies have later '):
- see_oudated_deps = True
- raw_report.close()
- return outdated_deps
- except:
- raise
-
-
-def extract_single_dep(dep):
- """
- Extract a single dependency check record from Java and Python reports.
- Args:
- dep: e.g " - org.assertj:assertj-core [2.5.0 -> 3.10.0]".
- Return:
- dependency name, current version, latest version.
- """
- pattern = " - ([\s\S]*)\[([\s\S]*) -> ([\s\S]*)\]"
- match = re.match(pattern, dep)
- if match is None:
- raise InvalidFormatError("Failed to extract the dependency information: {}".format(dep))
- return match.group(1).strip(), match.group(2).strip(), match.group(3).strip()
-
-
-def prioritize_dependencies(deps, sdk_type):
- """
- Extracts and analyze dependency versions and release dates.
- Returns a collection of dependencies which is "high priority" in html format:
- 1. dependency has major release. e.g org.assertj:assertj-core [2.5.0 -> 3.10.0]
- 2. dependency is 3 sub-versions behind the newest one. e.g org.tukaani:xz [1.5 -> 1.8]
- 3. dependency has not been updated for more than 6 months.
-
- Args:
- deps: A collection of outdated dependencies.
- Return:
- high_priority_deps: A collection of dependencies which need to be taken care of before next release.
- """
-
- project_id = ReportGeneratorConfig.GCLOUD_PROJECT_ID
- dataset_id = ReportGeneratorConfig.DATASET_ID
- table_id = ReportGeneratorConfig.get_bigquery_table_id(sdk_type)
- high_priority_deps = []
- bigquery_client = BigQueryClientUtils(project_id, dataset_id, table_id)
-
- for dep in deps:
- try:
- if re.match(r'https?://', dep.lstrip()):
- # Gradle-version-plugin's output contains URLs of the libraries
- continue
- logging.info("\n\nStart processing: " + dep)
- dep_name, curr_ver, latest_ver = extract_single_dep(dep)
- curr_release_date = None
- latest_release_date = None
- group_id = None
-
- if sdk_type == 'Java':
- # extract the groupid and artifactid
- group_id, artifact_id = dep_name.split(":")
- dep_details_url = "{0}/{1}/{2}".format(ReportGeneratorConfig.MAVEN_CENTRAL_URL, group_id, artifact_id)
- curr_release_date = find_release_time_from_maven_central(group_id, artifact_id, curr_ver)
- latest_release_date = find_release_time_from_maven_central(group_id, artifact_id, latest_ver)
- else:
- dep_details_url = ReportGeneratorConfig.PYPI_URL + dep_name
- curr_release_date = find_release_time_from_python_compatibility_checking_service(dep_name, curr_ver)
- latest_release_date = find_release_time_from_python_compatibility_checking_service(dep_name, latest_ver)
-
- if not curr_release_date or not latest_release_date:
- curr_release_date, latest_release_date = query_dependency_release_dates_from_bigquery(bigquery_client,
- dep_name,
- curr_ver,
- latest_ver)
- dep_info = """
- {1} |
- {2} |
- {3} |
- {4} |
- {5} | """.format(dep_details_url,
- dep_name,
- curr_ver,
- latest_ver,
- curr_release_date,
- latest_release_date)
- if (version_comparer.compare_dependency_versions(curr_ver, latest_ver) or
- compare_dependency_release_dates(curr_release_date, latest_release_date)):
- high_priority_deps.append(dep_info)
-
- except:
- traceback.print_exc()
- continue
-
- bigquery_client.clean_stale_records_from_table()
- return high_priority_deps
-
-
-def find_release_time_from_maven_central(group_id, artifact_id, version):
- """
- Find release dates from Maven Central REST API.
- Args:
- group_id:
- artifact_id:
- version:
- Return:
- release date
- """
- url = "http://search.maven.org/solrsearch/select?q=g:{0}+AND+a:{1}+AND+v:{2}".format(
- group_id,
- artifact_id,
- version
- )
- logging.info('Finding release date of {0}:{1} {2} from the Maven Central'.format(
- group_id,
- artifact_id,
- version
- ))
- try:
- response = request_session_with_retries().get(url)
- if not response.ok:
- logging.error("""Failed finding the release date of {0}:{1} {2}.
- The response status code is not ok: {3}""".format(group_id,
- artifact_id,
- version,
- str(response.status_code)))
- return None
- response_data = response.json()
- release_timestamp_mills = response_data['response']['docs'][0]['timestamp']
- release_date = datetime.fromtimestamp(release_timestamp_mills/1000).date()
- return release_date
- except Exception as e:
- logging.error("Errors while extracting the release date: " + str(e))
- return None
-
-
-def find_release_time_from_python_compatibility_checking_service(dep_name, version):
- """
- Query release dates by using Python compatibility checking service.
- Args:
- dep_name:
- version:
- Return:
- release date
- """
- url = 'http://104.197.8.72/?package={0}=={1}&python-version=2'.format(
- dep_name,
- version
- )
- logging.info('Finding release time of {0} {1} from the python compatibility checking service.'.format(
- dep_name,
- version
- ))
- try:
- response = request_session_with_retries().get(url)
- if not response.ok:
- logging.error("""Failed finding the release date of {0} {1}.
- The response status code is not ok: {2}""".format(dep_name,
- version,
- str(response.status_code)))
- return None
- response_data = response.json()
- release_datetime = response_data['dependency_info'][dep_name]['installed_version_time']
- release_date = datetime.strptime(release_datetime, '%Y-%m-%dT%H:%M:%S').date()
- return release_date
- except Exception as e:
- logging.error("Errors while extracting the release date: " + str(e))
- return None
-
-
-def request_session_with_retries():
- """
- Create an http session with retries
- """
- session = requests.Session()
- retries = Retry(total=3)
- session.mount('http://', HTTPAdapter(max_retries=retries))
- session.mount('https://', HTTPAdapter(max_retries=retries))
- return session
-
-
-def query_dependency_release_dates_from_bigquery(bigquery_client, dep_name, curr_ver_in_beam, latest_ver):
- """
- Query release dates of current version and the latest version from BQ tables.
- Args:
- bigquery_client: a bq client object that bundle configurations for API requests
- dep_name: dependency name
- curr_ver_in_beam: the current version used in beam
- latest_ver: the later version
- Return:
- A tuple that contains `curr_release_date` and `latest_release_date`.
- """
- try:
- curr_release_date, is_currently_used_bool = bigquery_client.query_dep_info_by_version(dep_name, curr_ver_in_beam)
- latest_release_date, _ = bigquery_client.query_dep_info_by_version(dep_name, latest_ver)
- date_today = datetime.today().date()
-
- # sync to the bigquery table on the dependency status of the currently used version.
- if not is_currently_used_bool:
- currently_used_version_in_db, currently_used_release_date_in_db = bigquery_client.query_currently_used_dep_info_in_db(dep_name)
- if currently_used_version_in_db is not None:
- bigquery_client.delete_dep_from_table(dep_name, currently_used_version_in_db)
- bigquery_client.insert_dep_to_table(dep_name, currently_used_version_in_db, currently_used_release_date_in_db, is_currently_used=False)
- if curr_release_date is None:
- bigquery_client.insert_dep_to_table(dep_name, curr_ver_in_beam, date_today, is_currently_used=True)
- else:
- bigquery_client.delete_dep_from_table(dep_name, curr_ver_in_beam)
- bigquery_client.insert_dep_to_table(dep_name, curr_ver_in_beam, curr_release_date, is_currently_used=True)
- # sync to the bigquery table on the dependency status of the latest version.
- if latest_release_date is None:
- bigquery_client.insert_dep_to_table(dep_name, latest_ver, date_today, is_currently_used=False)
- latest_release_date = date_today
- except Exception:
- raise
- return curr_release_date, latest_release_date
-
-
-def compare_dependency_release_dates(curr_release_date, latest_release_date):
- """
- Compare release dates of current using version and the latest version.
- Return true if the current version is behind over 60 days.
- Args:
- curr_release_date
- latest_release_date
- Return:
- boolean
- """
- if not curr_release_date or not latest_release_date:
- return False
- else:
- if (latest_release_date - curr_release_date).days >= ReportGeneratorConfig.MAX_STALE_DAYS:
- return True
- return False
-
-
-def generate_report(sdk_type):
- """
- Write SDK dependency check results into a html report.
- Args:
- sdk_type: String [Java, Python, TODO: Go]
- """
- report_name = ReportGeneratorConfig.FINAL_REPORT
- raw_report = ReportGeneratorConfig.get_raw_report(sdk_type)
-
- if os.path.exists(report_name):
- append_write = 'a'
- else:
- append_write = 'w'
-
- try:
- # Extract dependency check results from build/dependencyUpdate
- report = open(report_name, append_write)
- if os.path.isfile(raw_report):
- outdated_deps = extract_results(raw_report)
- else:
- report.write("Did not find the raw report of dependency check: {}".format(raw_report))
- report.close()
- return
-
- # Prioritize dependencies by comparing versions and release dates.
- high_priority_deps = prioritize_dependencies(outdated_deps, sdk_type)
-
- # Write results to a report
- subtitle = "High Priority Dependency Updates Of Beam {} SDK:
\n".format(sdk_type)
- table_fields = """
- {0} |
- {1} |
- {2} |
- {3} |
- {4} |
-
""".format("Dependency Name",
- "Current Version",
- "Latest Version",
- "Release Date Of the Current Used Version",
- "Release Date Of The Latest Release")
- report.write(subtitle)
- report.write("\n")
- report.write(table_fields)
- for dep in high_priority_deps:
- report.write("%s" % dep)
- report.write("
\n")
- except Exception as e:
- traceback.print_exc()
- logging.error("Failed generate the dependency report. " + str(e))
- report.write(' {0}
'.format(str(e)))
-
- report.close()
- logging.info("Dependency check on {0} SDK complete. The report is created.".format(sdk_type))
-
-def main(args):
- """
- Main method.
- Args:
- args[0]: type of the check [Java, Python]
- """
- generate_report(args[0])
-
-
-if __name__ == '__main__':
- main(sys.argv[1:])
diff --git a/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py b/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py
deleted file mode 100644
index b7b2ec092a4c2..0000000000000
--- a/.test-infra/jenkins/dependency_check/dependency_check_report_generator_test.py
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/usr/bin/env python
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This script performs testing of scenarios from verify_performance_test_results.py
-#
-
-from __future__ import print_function
-import unittest
-from mock import patch, mock_open
-from datetime import datetime
-from .dependency_check_report_generator import prioritize_dependencies
-
-
-_PROJECT_ID = 'mock-apache-beam-testing'
-_DATASET_ID = 'mock-beam_dependency_states'
-_TABLE_ID = 'mock-java_dependency_states'
-_SDK_TYPE = 'Java'
-
-# initialize current/latest version release dates for low-priority (LP) and high-priority (HP) dependencies
-_LP_CURR_VERSION_DATE = datetime.strptime('2000-01-01', '%Y-%m-%d')
-_LATEST_VERSION_DATE = datetime.strptime('2000-01-02', '%Y-%m-%d')
-_HP_CURR_VERSION_DATE = datetime.strptime('1999-01-01', '%Y-%m-%d')
-_MOCKED_OWNERS_FILE = "deps: "
-
-
-@patch('google.cloud.bigquery.Client')
-@patch('dependency_check.bigquery_client_utils.BigQueryClientUtils.clean_stale_records_from_table')
-class DependencyCheckReportGeneratorTest(unittest.TestCase):
- """Tests for `dependency_check_report_generator.py`."""
-
- def setUp(self):
- print("\n\nTest : " + self._testMethodName)
-
-
- @patch('dependency_check.bigquery_client_utils.BigQueryClientUtils')
- def test_empty_dep_input(self, *args):
- """
- Test on empty outdated dependencies.
- Expect: empty report
- """
- with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)):
- report = prioritize_dependencies([], _SDK_TYPE)
- self.assertEqual(len(report), 0)
-
-
- @patch('dependency_check.dependency_check_report_generator.find_release_time_from_maven_central',
- side_effect = [_LP_CURR_VERSION_DATE, _LATEST_VERSION_DATE,
- _LP_CURR_VERSION_DATE, _LATEST_VERSION_DATE,
- _HP_CURR_VERSION_DATE, _LATEST_VERSION_DATE,
- _LP_CURR_VERSION_DATE, _LATEST_VERSION_DATE,])
- def test_normal_dep_input(self, *args):
- """
- Test on a normal outdated dependencies set.
- Expect: group1:artifact1, group2:artifact2, and group3:artifact3
- """
- deps = [
- " - group1:artifact1 [1.0.0 -> 3.0.0]",
- " - group2:artifact2 [1.0.0 -> 1.3.0]",
- " - group3:artifact3 [1.0.0 -> 1.1.0]",
- " - group4:artifact4 [1.0.0 -> 1.1.0]"
- ]
- with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)):
- report = prioritize_dependencies(deps, _SDK_TYPE)
- self.assertEqual(len(report), 3)
- self.assertIn('group1:artifact1', report[0])
- self.assertIn('group2:artifact2', report[1])
- self.assertIn('group3:artifact3', report[2])
-
-
- @patch('dependency_check.dependency_check_report_generator.find_release_time_from_maven_central',
- side_effect = [_LP_CURR_VERSION_DATE,
- _LATEST_VERSION_DATE,])
- def test_dep_with_nondigit_major_versions(self, *args):
- """
- Test on a outdated dependency with non-digit major number.
- Expect: group1:artifact1
- """
- deps = [" - group1:artifact1 [Release1-123 -> Release2-456]"]
- with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)):
- report = prioritize_dependencies(deps, _SDK_TYPE)
- self.assertEqual(len(report), 1)
- self.assertIn('group1:artifact1', report[0])
-
-
- @patch('dependency_check.dependency_check_report_generator.find_release_time_from_maven_central',
- side_effect = [_LP_CURR_VERSION_DATE,
- _LATEST_VERSION_DATE,])
- def test_dep_with_nondigit_minor_versions(self, *args):
- """
- Test on a outdated dependency with non-digit minor number.
- Expect: group1:artifact1
- """
- deps = [" - group1:artifact1 [0.rc1.0 -> 0.rc2.0]"]
- with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)):
- report = prioritize_dependencies(deps, _SDK_TYPE)
- self.assertEqual(len(report), 1)
- self.assertIn('group1:artifact1', report[0])
-
-
- @patch('dependency_check.dependency_check_report_generator.find_release_time_from_maven_central',
- side_effect = [_HP_CURR_VERSION_DATE,_LATEST_VERSION_DATE,])
- def test_invalid_dep_input(self, *args):
- """
- Test on a invalid outdated dependencies format.
- Expect: Exception through out. And group2:artifact2 is picked.
- """
- deps = [
- "- group1:artifact1 (1.0.0, 2.0.0)",
- " - group2:artifact2 [1.0.0 -> 2.0.0]"
- ]
- with patch('builtins.open', mock_open(read_data=_MOCKED_OWNERS_FILE)):
- report = prioritize_dependencies(deps, _SDK_TYPE)
- self.assertEqual(len(report), 1)
- self.assertIn('group2:artifact2', report[0])
-
-
-if __name__ == '__main__':
- unittest.main()
-
diff --git a/.test-infra/jenkins/dependency_check/generate_report.sh b/.test-infra/jenkins/dependency_check/generate_report.sh
deleted file mode 100755
index cfb66ec951995..0000000000000
--- a/.test-infra/jenkins/dependency_check/generate_report.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This script will be run by Jenkins as a Python dependency test.
-
-set -e
-set -v
-
-# Get currently used Python version from args or assume a default.
-PYTHON=${1:-python3}
-
-REPORT_DESCRIPTION="
- A dependency update is high priority if it satisfies one of following criteria:
-
-- It has major versions update available, e.g. org.assertj:assertj-core 2.5.0 -> 3.10.0;
-
-
-- It is over 3 minor versions behind the latest version, e.g. org.tukaani:xz 1.5 -> 1.8;
-
-
-- The current version is behind the later version for over 180 days, e.g. com.google.auto.service:auto-service 2014-10-24 -> 2017-12-11.
-
- In Beam, we make a best-effort attempt at keeping all dependencies up-to-date.
- In the future, issues will be filed and tracked for these automatically,
- but in the meantime you can search for existing issues or open a new one.
-
-"
-
-
-# Virtualenv for the rest of the script to run setup
-$PYTHON -m venv dependency/check
-
-. ./dependency/check/bin/activate
-pip install --upgrade pip setuptools wheel
-pip install --upgrade google-cloud-bigquery google-cloud-bigtable google-cloud-core
-rm -f build/dependencyUpdates/beam-dependency-check-report.txt
-
-# Install packages and run the unit tests of the report generator
-pip install mock pyyaml
-cd $WORKSPACE/src/.test-infra/jenkins
-$PYTHON -m dependency_check.dependency_check_report_generator_test
-$PYTHON -m dependency_check.version_comparer_test
-
-echo "" > $WORKSPACE/src/build/dependencyUpdates/beam-dependency-check-report.html
-
-$PYTHON -m dependency_check.dependency_check_report_generator Python
-
-$PYTHON -m dependency_check.dependency_check_report_generator Java
-
-echo "$REPORT_DESCRIPTION " >> $WORKSPACE/src/build/dependencyUpdates/beam-dependency-check-report.html
diff --git a/.test-infra/jenkins/dependency_check/report_generator_config.py b/.test-infra/jenkins/dependency_check/report_generator_config.py
deleted file mode 100644
index 18396ee43f8fd..0000000000000
--- a/.test-infra/jenkins/dependency_check/report_generator_config.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Defines constants and helper methods used by the dependency_report_generator
-
-import os
-
-class ReportGeneratorConfig:
-
- # Jenkins Working Space
- WORKING_SPACE = os.environ['WORKSPACE']
-
- # Constants for dependency prioritization
- GCLOUD_PROJECT_ID = 'apache-beam-testing'
- DATASET_ID = 'beam_dependency_states'
- PYTHON_DEP_TABLE_ID = 'python_dependency_states'
- JAVA_DEP_TABLE_ID = 'java_dependency_states'
- PYTHON_DEP_RAW_REPORT = WORKING_SPACE + '/src/build/dependencyUpdates/python_dependency_report.txt'
- JAVA_DEP_RAW_REPORT = WORKING_SPACE + '/src/build/dependencyUpdates/report.txt'
- FINAL_REPORT = WORKING_SPACE + '/src/build/dependencyUpdates/beam-dependency-check-report.html'
- MAX_STALE_DAYS = 360
- MAX_MINOR_VERSION_DIFF = 3
- PYPI_URL = "https://pypi.org/project/"
- MAVEN_CENTRAL_URL = "https://mvnrepository.com/artifact"
-
- # Dependency Owners
- JAVA_DEP_OWNERS = WORKING_SPACE + '/src/ownership/JAVA_DEPENDENCY_OWNERS.yaml'
- PYTHON_DEP_OWNERS = WORKING_SPACE + '/src/ownership/PYTHON_DEPENDENCY_OWNERS.yaml'
-
-
- @classmethod
- def get_bigquery_table_id(cls, sdk_type):
- if sdk_type.lower() == 'java':
- return cls.JAVA_DEP_TABLE_ID
- elif sdk_type.lower() == 'python':
- return cls.PYTHON_DEP_TABLE_ID
- else:
- raise UndefinedSDKTypeException("""Undefined SDK Type: {0}.
- Could not find the BigQuery table for {1} dependencies.""".format(sdk_type, sdk_type))
-
-
- @classmethod
- def get_raw_report(cls, sdk_type):
- if sdk_type.lower() == 'java':
- return cls.JAVA_DEP_RAW_REPORT
- elif sdk_type.lower() == 'python':
- return cls.PYTHON_DEP_RAW_REPORT
- else:
- raise UndefinedSDKTypeException("""Undefined SDK Type: {0}.
- Could not find the dependency reports for the {1} SDK.""".format(sdk_type, sdk_type))
-
-
- @classmethod
- def get_owners_file(cls, sdk_type):
- if sdk_type.lower() == 'java':
- return cls.JAVA_DEP_OWNERS
- elif sdk_type.lower() == 'python':
- return cls.PYTHON_DEP_OWNERS
- else:
- raise UndefinedSDKTypeException("""Undefined SDK Type: {0}.
- Could not find the Owners file for the {1} SDK.""".format(sdk_type, sdk_type))
-
-
-class UndefinedSDKTypeException(Exception):
- """Indicates an error has occurred in while reading constants."""
-
- def __init__(self, msg):
- super(UndefinedSDKTypeException, self).__init__(msg)
diff --git a/.test-infra/jenkins/dependency_check/version_comparer.py b/.test-infra/jenkins/dependency_check/version_comparer.py
deleted file mode 100644
index d8134ecf11d4f..0000000000000
--- a/.test-infra/jenkins/dependency_check/version_comparer.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from dependency_check.report_generator_config import ReportGeneratorConfig
-
-def compare_dependency_versions(curr_ver, latest_ver):
- """
- Compare the current using version and the latest version.
- Return true if a major version change was found, or 3 minor versions that the current version is behind.
- Args:
- curr_ver
- latest_ver
- Return:
- boolean
- """
- if curr_ver is None or latest_ver is None:
- return True
- else:
- curr_ver_splitted = curr_ver.split('.')
- latest_ver_splitted = latest_ver.split('.')
- curr_major_ver = curr_ver_splitted[0]
- latest_major_ver = latest_ver_splitted[0]
- # compare major versions
- if curr_major_ver != latest_major_ver:
- return True
- # compare minor versions
- else:
- curr_minor_ver = curr_ver_splitted[1] if len(curr_ver_splitted) > 1 else None
- latest_minor_ver = latest_ver_splitted[1] if len(latest_ver_splitted) > 1 else None
- if curr_minor_ver is not None and latest_minor_ver is not None:
- if (not curr_minor_ver.isdigit() or not latest_minor_ver.isdigit()) and curr_minor_ver != latest_minor_ver:
- return True
- elif int(curr_minor_ver) + ReportGeneratorConfig.MAX_MINOR_VERSION_DIFF <= int(latest_minor_ver):
- return True
- # TODO: Comparing patch versions if needed.
- return False
diff --git a/.test-infra/jenkins/dependency_check/version_comparer_test.py b/.test-infra/jenkins/dependency_check/version_comparer_test.py
deleted file mode 100644
index b37fb5f607a7d..0000000000000
--- a/.test-infra/jenkins/dependency_check/version_comparer_test.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from . import version_comparer
-import unittest
-
-class VersionComparerTest(unittest.TestCase):
- """Tests for `version_comparer.py`."""
-
- def setUp(self):
- print("\n\nTest : " + self._testMethodName)
-
- def test_compare_major_verison_true(self):
- curr_ver = '1.0.0'
- latest_ver = '2.0.0'
- self.assertTrue(version_comparer.compare_dependency_versions(curr_ver, latest_ver))
-
- def test_compare_minor_version_true(self):
- curr_ver = '1.0.0'
- latest_ver = '1.3.0'
- self.assertTrue(version_comparer.compare_dependency_versions(curr_ver, latest_ver))
-
- def test_compare_non_semantic_version_true(self):
- curr_ver = '1.rc1'
- latest_ver = '1.rc2'
- self.assertTrue(version_comparer.compare_dependency_versions(curr_ver, latest_ver))
-
- def test_compare_minor_version_false(self):
- curr_ver = '1.0.0'
- latest_ver = '1.2.0'
- self.assertFalse(version_comparer.compare_dependency_versions(curr_ver, latest_ver))
-
- def test_compare_same_version_false(self):
- curr_ver = '1.0.0'
- latest_ver = '1.0.0'
- self.assertFalse(version_comparer.compare_dependency_versions(curr_ver, latest_ver))
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/.test-infra/jenkins/job_Dependency_Check.groovy b/.test-infra/jenkins/job_Dependency_Check.groovy
deleted file mode 100644
index b80f2610314db..0000000000000
--- a/.test-infra/jenkins/job_Dependency_Check.groovy
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-
-job('beam_Dependency_Check') {
- description('Runs Beam dependency check.')
-
- // Set common parameters.
- commonJobProperties.setTopLevelMainJobProperties(
- delegate, 'master', 100, true, 'beam', false)
-
- // Allows triggering this build against pull requests.
- commonJobProperties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Beam Dependency Check',
- 'Run Dependency Check',
- false
- )
-
- commonJobProperties.setAutoJob(
- delegate,
- '@weekly')
-
- steps {
- gradle {
- rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks('runBeamDependencyCheck')
- commonJobProperties.setGradleSwitches(delegate)
- switches('-Drevision=release')
- }
-
- shell('cd ' + commonJobProperties.checkoutDir +
- ' && bash .test-infra/jenkins/dependency_check/generate_report.sh ' +
- commonJobProperties.PYTHON)
- }
-
- def date = new Date().format('yyyy-MM-dd')
- publishers {
- extendedEmail {
- triggers {
- always {
- recipientList('dev@beam.apache.org')
- contentType('text/html')
- subject("Beam Dependency Check Report (${date})")
- content('''${FILE, path="src/build/dependencyUpdates/beam-dependency-check-report.html"}''')
- }
- }
- }
- archiveArtifacts {
- pattern('src/build/dependencyUpdates/beam-dependency-check-report.html')
- onlyIfSuccessful()
- }
- wsCleanup {
- excludePattern('src/build/dependencyUpdates/beam-dependency-check-report.html')
- }
- }
-}
diff --git a/build.gradle.kts b/build.gradle.kts
index b89270a9537fe..77520c3ce7233 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -18,11 +18,6 @@
plugins {
base
- // This plugin provides a task to determine which dependencies have updates.
- // Additionally, the plugin checks for updates to Gradle itself.
- //
- // See https://github.com/ben-manes/gradle-versions-plugin for further details.
- id("com.github.ben-manes.versions") version "0.33.0"
// Apply one top level rat plugin to perform any required license enforcement analysis
id("org.nosphere.apache.rat") version "0.8.0"
// Enable gradle-based release management
@@ -580,11 +575,6 @@ tasks.register("javaExamplesDataflowPrecommit") {
dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:preCommit")
}
-tasks.register("runBeamDependencyCheck") {
- dependsOn(":dependencyUpdates")
- dependsOn(":sdks:python:dependencyUpdates")
-}
-
tasks.register("whitespacePreCommit") {
// TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version.
dependsOn(":sdks:python:test-suites:tox:py38:archiveFilesToLint")
diff --git a/ownership/JAVA_DEPENDENCY_OWNERS.yaml b/ownership/JAVA_DEPENDENCY_OWNERS.yaml
index 063b2bcc145fd..eb43b8a49fd68 100644
--- a/ownership/JAVA_DEPENDENCY_OWNERS.yaml
+++ b/ownership/JAVA_DEPENDENCY_OWNERS.yaml
@@ -144,11 +144,6 @@ deps:
artifact: jackson-module-scala_2.11
owners:
- com.github.ben-manes:gradle-versions-plugin:
- group: com.github.ben-manes
- artifact: gradle-versions-plugin
- owners:
-
com.github.jengelman.gradle.plugins:shadow:
group: com.github.jengelman.gradle.plugins
artifact: shadow
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 7dac7c2ec0ff6..0c2ed72dfba18 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -134,11 +134,6 @@ def depSnapshot = tasks.register("depSnapshot") {
}
}
-tasks.register("dependencyUpdates") {
- dependsOn ':dependencyUpdates'
- dependsOn ':sdks:python:test-suites:tox:pycommon:dependency-check'
-}
-
tasks.register("buildSnapshot") {
dependsOn sdist
dependsOn depSnapshot
diff --git a/sdks/python/scripts/run_dependency_check.sh b/sdks/python/scripts/run_dependency_check.sh
deleted file mode 100755
index d60d6016fd53b..0000000000000
--- a/sdks/python/scripts/run_dependency_check.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This script will be run by Jenkins as a Python dependency test.
-
-set -euv
-
-mkdir -p $WORKSPACE/src/build/dependencyUpdates
-rm -f $WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt
-
-# List all outdated dependencies and write results in pythonDependencyReport
-echo "The following dependencies have later release versions:" > $WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt
-pip list --outdated | sed -n '1,2!p' | while IFS= read -r line
-do
- echo $line | while IFS=' ' read dep curr_ver new_ver type
- do
- echo $line
- echo " - $dep [$curr_ver -> $new_ver]" >> $WORKSPACE/src/build/dependencyUpdates/python_dependency_report.txt
- done
-done
diff --git a/sdks/python/test-suites/tox/pycommon/build.gradle b/sdks/python/test-suites/tox/pycommon/build.gradle
index 7e697ec367e15..7a44c9eb92f01 100644
--- a/sdks/python/test-suites/tox/pycommon/build.gradle
+++ b/sdks/python/test-suites/tox/pycommon/build.gradle
@@ -28,6 +28,4 @@ toxTask "docs", "py38-docs"
assemble.dependsOn docs
task preCommitPyCommon() {
-}
-
-toxTask "dependency-check", "py3-dependency-check"
+}
\ No newline at end of file
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 50a883e26a7a6..9088c1483906f 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -244,16 +244,6 @@ commands =
yapf --version
time yapf --diff --parallel --recursive apache_beam
-[testenv:py3-dependency-check]
-# TODO(https://github.com/apache/beam/issues/20337): botocore, a part of [aws], wants docutils<0.16, but Sphinx
-# pulls in the latest docutils. Uncomment this line once botocore does not
-# conflict with Sphinx:
-# extras = docs,test,gcp,aws,interactive,interactive_test
-extras = test,gcp,aws,dataframe,interactive,interactive_test
-passenv = WORKSPACE
-commands =
- time {toxinidir}/scripts/run_dependency_check.sh
-
[testenv:jest]
setenv =
deps =