export 20 months of internal changes back to fsqio (#73)

It has been a hot minute since we last pushed updates from our internal repo to fsqio. In the intervening time, our typical release process has decayed to the point of no longer functioning and there are some major changes to how we build and manage code underway internally at Foursquare. This is a manual export of our internal changes back to fsqio covering roughly the last 20 months, bringing us current to [cd146cb](foursquare/foursquare.web@cd146cb) in the internal monorepo. Changes included here (along with contributing authors): - get `./pants test test/jvm::` passing again (@jvandew) - cleanups to rogue code, tests, & readmes (@jvandew) - upgrade junit to 4.13.2 (@jonshea) - upgrades to our python env & code in preparation for python 3 (@TansyArron, @OniOni, @jeffreyc) - cleanups & improvements to spindle ide-gen (@iantabolt, @ronbrz, @kylefowler) - shard key aware `save` operations in rogue with spindle & lift (@alilewin) - support for mypi interface stubs (@jonshea) - `explain` query support in rogue (@aetropolski) - bump scala minor version to 2.11.12 (@jvandew) - support for lazy mongo client init with rogue (@nfolkert-foursquare) - numerous other small improvements & bug fixes
foursquare · Jun 28, 2022 · f5a5699 · f5a5699
1 parent da7a3b8
commit f5a5699
Show file tree

Hide file tree

Showing 184 changed files with 2,899 additions and 2,661 deletions.
diff --git a/3rdparty/BUILD b/3rdparty/BUILD
@@ -7,7 +7,7 @@ jar_library(
     'opensource',
   ],
   jars = [
-    jar(org = 'io.sgr', name = 's2-geometry-library-java', rev = '1.0.0')
+    jar(org = 'io.sgr', name = 's2-geometry-library-java', rev = '1.0.1'),
   ]
 )
 

diff --git a/3rdparty/BUILD.opensource b/3rdparty/BUILD.opensource
@@ -1,6 +1,6 @@
 # Copyright 2015 Foursquare Labs Inc. All Rights Reserved.
 
-scala_version = '2.11.8'
+scala_version = '2.11.12'
 HADOOP_VERSION = '2.8.5'
 
 
@@ -55,6 +55,7 @@ managed_jar_dependencies(
     ':mongodb-driver',
     ':mongodb-driver-reactivestreams',
     ':netty',
+    ':netty4',
     ':janino-compiler',
     ':reflections',
     ':scalacheck',
@@ -542,30 +543,6 @@ jar_library(
   ],
 )
 
-# A new version of jackson with different org and fully qualified class names. It won't conflict with other jackson.
-# The reason for two jacksons. The former is UNUSED, but is necessary for using Finatra.
-# 2.9.10 breaks EMR usage. 2.6.7 breaks finatra.
-# Jackson is currently pinned meaning we can only have 1 version of it in our repo.
-# The goal is to pin 2.9.10 somehow without breaking EMR.
-# This is a placeholder for now and to allow for development of finatra services
-# while the dependency issue is being resolved. (@jwlodarczyk)
-# jackson_finatra_rev = '2.9.10'
-# jar_library(
-#   name = 'jackson-scala',
-#   jars = [
-#     jar(org = 'com.fasterxml.jackson.core', name = 'jackson-annotations', rev = jackson_finatra_rev),
-#     jar(org = 'com.fasterxml.jackson.core', name = 'jackson-core', rev = jackson_finatra_rev),
-#     jar(org = 'com.fasterxml.jackson.core', name = 'jackson-databind', rev = '2.9.10.1'),
-#     # NOTE(iant): 2.6.7.1 for 2.12 support; should be jackson_rev in the future
-#     scala_jar(org = 'com.fasterxml.jackson.module', name = 'jackson-module-scala', rev = jackson_finatra_rev),
-#     jar(org = 'com.fasterxml.jackson.datatype', name = 'jackson-datatype-joda', rev = jackson_finatra_rev),
-#     jar(org = 'com.fasterxml.jackson.datatype', name = 'jackson-datatype-guava', rev = jackson_finatra_rev),
-#     jar(org = 'com.fasterxml.jackson.datatype', name = 'jackson-datatype-jdk8', rev = jackson_finatra_rev),
-#     jar(org = 'com.fasterxml.jackson.datatype', name = 'jackson-datatype-jsr310', rev = jackson_finatra_rev),
-#     jar(org = 'com.fasterxml.jackson.dataformat', name = 'jackson-dataformat-cbor', rev = jackson_finatra_rev),
-#   ],
-# )
-
 jackson_rev = '2.6.7'
 jar_library(
   name = 'jackson-scala',
@@ -706,8 +683,7 @@ jar_library(
     'fscommon',
   ],
   jars = [
-    jar(org = 'junit', name = 'junit', rev = '4.12'),
-    jar(org = 'com.novocode', name = 'junit-interface', rev = '0.11'),
+    jar(org = 'junit', name = 'junit', rev = '4.13.2'),
   ],
 )
 
@@ -1039,10 +1015,54 @@ jar_library(
   ]
 )
 
+    # NOTE(jacob): Depending on the monolithic netty jar rather than its individual
+    #   modules causes all kinds of headaches with other libraries that try to do the
+    #   right thing and only pull in what they need. There is some talk in the netty
+    #   project of making the monolith an empty wrapper that pulls in all the module jars
+    #   (see https://github.com/netty/netty/issues/4671 for details), but for now it
+    #   bundles duplicate classes and we have to sort out the conflict. Our dep report
+    #   lists spark as the only problem child pulling this in, but just one is enough.
+    #
+    #   What we do here is exclude the monolith, and instead declare a jar_library
+    #   containing the individual netty modules we need. If you add a new 3rdparty jar
+    #   with a dependency on netty-all, you may see classloading issues if it uses code
+    #   from a module we aren't pulling in. It is up to you to identify the missing
+    #   module, add it to our netty4 jar_library, and then add a dependency from your
+    #   new jar_library to our netty4 jar_library.
+    #
+    #   This also holds for libraries depending on netty modules we haven't yet declared.
+    #   We will transitively pull them in, but they won't be pinned at the proper version
+    #   and you may see classloading issues until you add them to this jar_library.
+    #
+    #   Note also included on the netty-all jar exclude in pants.ini.
+netty4_rev = '4.1.43.Final'
 jar_library(
   name = 'netty4',
+  tags = [
+    'opensource',
+    'fscommon',
+  ],
   jars = [
-    jar(org = 'io.netty', name = 'netty-all', rev = '4.1.45.Final'),
+    jar(org = 'io.netty', name = 'netty-buffer', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-codec', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-codec-dns', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-codec-http', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-codec-http2', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-codec-socks', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-common', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-handler', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-handler-proxy', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-resolver', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-resolver-dns', rev = netty4_rev),
+    jar(org = 'io.netty', name = 'netty-tcnative-boringssl-static', rev = '2.0.26.Final'),
+    jar(org = 'io.netty', name = 'netty-transport', rev = netty4_rev),
+    jar(
+      org = 'io.netty',
+      name = 'netty-transport-native-epoll',
+      classifier = 'linux-x86_64',
+      rev = netty4_rev,
+    ),
+    jar(org = 'io.netty', name = 'netty-transport-native-unix-common', rev = netty4_rev),
   ],
   managed_dependencies=':fsqio-management',
 )

diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt
@@ -1,6 +1,7 @@
 ansicolors==1.0.2
 attrs<19.2.0                # Pinned to avoid pulling in unsafe upgrade
 configparser==3.5.0
+coverage>=4.5,<4.6        # Pants dep - fix resolution conflicts by matching Pants version.
 future==0.16.0
 mock==2.0.0
 more-itertools==5.0.0       # Pinned to avoid pulling in unsafe upgrade
@@ -13,6 +14,9 @@ pantsbuild.pants.testinfra==1.7.0
 pex==1.3.2                  # Pants dep - fix resolution conflicts by matching Pants version.
 psycopg2==2.7
 pymongo==2.8
+pytest-cov>=2.5,<2.6       # Pants dep - fix resolution conflicts by matching Pants version.
+pytest==3.6.4              # Keep this in sync with --pytest-requirements in pants.ini.
+pytest-timeout>=1.2,<1.3   # Pants dep - fix resolution conflicts by matching Pants version.
 pytz>=2016.6.1
 requests[security]>=2.8.14  # Pants dep - fix resolution conflicts by matching
 requests_futures>=0.9.4,<0.10

diff --git a/BUILD.opensource b/BUILD.opensource
@@ -1,6 +1,24 @@
 # Copyright 2016 Foursquare Labs Inc. All Rights Reserved.
 
-scala_version = '2.11.8'
+scala_version = '2.11.12'
+
+
+# NOTE(jacob): We have some tests for pants plugin functionality that need the pants
+#   runner script and a few core BUILD files available in the working directory. These
+#   will otherwise succeed locally, but fail when run under a chroot in ci unless we
+#   explicitly pull them in.
+files(
+  name = 'pants',
+  sources = [
+    '3rdparty/BUILD',
+    'BUILD.tools',
+    'pants',
+  ],
+  scope = "runtime",
+  tags = [
+    'opensource',
+  ]
+)
 
 
 # NOTE: Now that we have full use of Ivy and target scopes, this should probably be adjusted.

diff --git a/build-support/fsqio/ivy/fsqio.ivysettings.xml b/build-support/fsqio/ivy/fsqio.ivysettings.xml
@@ -33,11 +33,6 @@ Copyright 2015 Foursquare Labs Inc. All Rights Reserved.
                m2compatible="true"
                usepoms="true"
                root="https://maven.twttr.com/"/>
-      <!-- Publishes an s2 jar publicly. Thank you! -->
-      <ibiblio name="jnegre"
-               m2compatible="true"
-               usepoms="true"
-               root="https://bintray.com/artifact/download/jnegre/maven"/>
       <!-- Cascading specific maven repo -->
       <ibiblio name="conjars"
                m2compatible="true"

diff --git a/build-support/fsqio/mypy/mypy.ini b/build-support/fsqio/mypy/mypy.ini
@@ -5,6 +5,7 @@ show_column_numbers = True
 show_error_context = False
 scripts_are_modules = True
 allow_untyped_globals = True
+mypy_path = src/python/fsqio/pants/python/mypy_stubs
 
 [mypy-foursquare.*,fsqio.*]
 follow_imports = silent

diff --git a/build-support/fsqio/upkeep/libs/env.sh b/build-support/fsqio/upkeep/libs/env.sh
@@ -78,7 +78,7 @@ export PANTS_PYTHON_SETUP_WHEEL_VERSION="${PANTS_PYTHON_SETUP_WHEEL_VERSION:-$FS
 #   * '--test-pytest-prep-requirements': Accepts a list of additional requirements respected during resolve.
 
 export PANTS_PYTEST_REQUIREMENTS=${PANTS_PYTEST_REQUIREMENTS:-"pytest==$FS_PYTEST_VERS"}
-pytest_prep_defaults="['more-itertools<6.0.0', 'attrs<19.2.0', 'setuptools==$PANTS_PYTHON_SETUP_SETUPTOOLS_VERSION']"
+pytest_prep_defaults="['attrs<19.2.0', 'coverage>=4.5,<4.6', 'more-itertools<6.0.0', 'setuptools==$PANTS_PYTHON_SETUP_SETUPTOOLS_VERSION']"
 export PANTS_TEST_PYTEST_PREP_REQUIREMENTS=${PANTS_TEST_PYTEST_PREP_REQUIREMENTS:-$pytest_prep_defaults}
 export PANTS_PYTEST_COV_REQUIREMENTS=${PANTS_PYTEST_COV_REQUIREMENTS:-"pytest-cov>=2.5,<2.6"}
 

diff --git a/build-support/fsqio/upkeep/libs/fetcher.sh b/build-support/fsqio/upkeep/libs/fetcher.sh
@@ -28,7 +28,7 @@ function cached_download() {
     if [[ ${FSQ_RUNNING_ON_OSX} == "True" ]] && [[ ! -f ${FS_WGET_BINARY} ]]; then
       (curl "-fL${FSQ_CURL_PROGRESS}" ${FS_CURL_EXTRA_PARAMS} "${url}" >"${download_path}") || _fetch_failed="True"
     else
-      (${FS_WGET_BINARY} ${FS_WGET_EXTRA_PARAMS} -O ${download_path} ${url}) || _fetch_failed="True"
+      (${FS_WGET_BINARY} ${FS_WGET_EXTRA_PARAMS} -nc -O ${download_path} ${url}) || _fetch_failed="True"
     fi
     (
       [[ -z ${_fetch_failed} ]] &&

diff --git a/build-support/fsqio/upkeep/scripts/opensource-pants-env.sh b/build-support/fsqio/upkeep/scripts/opensource-pants-env.sh
@@ -9,9 +9,20 @@ if [[ -z "${BUILD_ROOT+x}" ]]; then
   exit 1
 fi
 
+function which_python {
+  found_python="$(which python2.7 || echo "")"
+  if [[ -z "${found_python}" ]]
+  then
+    echo 'fsqio requires python2.7, please install it or set the $PYTHON env variable to point at a valid python executable'
+    exit 1
+  else
+    echo "${found_python}"
+  fi
+}
+
 # Transitive song and dance in order to enforce priority order of CANONICAL_PYTHON -> PYTHON -> $(which python2.7)
 CANONICAL_PYTHON=${CANONICAL_PYTHON:-${PYTHON}}
-PYTHON=${CANONICAL_PYTHON:-$(which python2.7)}
+PYTHON=${CANONICAL_PYTHON:-$(which_python)}
 
 PANTS_BOOTSTRAP="${FSQIO_VENV_BOOTSTRAP}/bootstrap"
 

diff --git a/pants.ini b/pants.ini
@@ -21,7 +21,6 @@ pants_version: 1.7.0
 backend_packages: +[
     'pants.backend.docgen',
     'pants.contrib.confluence',
-    'pants.contrib.go',
     'pants.contrib.node',
     'fsqio.pants.buildgen.core',
     'fsqio.pants.buildgen.jvm',
@@ -40,7 +39,7 @@ backend_packages: +[
 
 # Mixed into all cache keys. Bump this to invalidate all existing artifacts.
 # We are using timestamps for cache keys. To bump it run `date +%Y-%m-%d:%H:%M:%S` and paste in the output.
-cache_key_gen_version: '2019-09-19:15:11:46'
+cache_key_gen_version: '2021-03-01:23:12:08'
 
 # The build_ignore_patterns uses .gitignore syntax.
 build_ignore: [
@@ -92,8 +91,6 @@ _base_jvm_options: [
 _dev_jvm_options: %(_base_jvm_options)s + [
   '-Xss8m',
   '-noverify',
-  '-Dmongo.connectionsPerHost=10',
-  '-Dmongo.threadsAllowedToBlockForConnectionMultiplier=10',
   ]
 
 # Overrides *all* options named "option" - this was the only effective way to override the defaults.
@@ -203,6 +200,7 @@ args: [
     '-S-language:reflectiveCalls',
     '-S-language:postfixOps',
     '-S-language:higherKinds',
+    '-S-target:jvm-1.8',
   ]
 incremental: True
 
@@ -234,14 +232,6 @@ async: True
 use_basename_prefix: True
 
 
-[go-distribution]
-version: 1.10.5
-
-
-[thrift.gen.go-thrift]
-version: 0.9.3
-
-
 [thrift]
 version: 0.9.1
 
@@ -320,7 +310,7 @@ test_dirs: [
     'test',
   ]
 
-target_whitelist: [
+target_allowlist: [
   'java_tests',
   'jsonc_config',
   'jsonc_library',
@@ -370,6 +360,9 @@ third_party_map: {
     'confluent_kafka': {
       'DEFAULT': '3rdparty/python:confluent-kafka',
     },
+    'enum': {
+      'DEFAULT': '3rdparty/python:enum34',
+    },
     'gen': {
       'apache': {
         'aurora': '3rdparty/python:apache.aurora.thrift',
@@ -384,7 +377,6 @@ third_party_map: {
       'contrib': {
         'confluence': '3rdparty/python:pantsbuild.pants.contrib.confluence',
         'node': '3rdparty/python:pantsbuild.pants.contrib.node',
-        'go': '3rdparty/python:pantsbuild.pants.contrib.go',
       },
     },
     'pants_test': {
@@ -405,9 +397,48 @@ third_party_map: {
 
 force_third_party: +['gnupg']
 
+
 [map-scala-library-used-addresses]
 third_party_map_file: src/resources/io/fsq/buildgen/third_party_map.ini
 
+
+[buildgen.publish]
+# Toggle on to enable provides support.
+skip: True
+
+opt_out_paths: [
+    # Opting out because testlibs should be proven useful to export.
+    # Requires adding `test/` to repo and metadata map to support.
+    'test/'
+  ]
+
+exportable_target_aliases: [
+    'java_library',
+    'scala_library',
+    'spindle_thrift_library',
+    'scala_record_library',
+  ]
+
+# NOTE(mateo): These use ExportableJvmTarget as a base and possibly could be published.
+# But doing so seems pathological. This disables even attempting it until the need is recognized.
+unexportable_target_aliases: [
+    'scala_js_library',
+    'scala_soy_library',
+    'junit_tests',
+  ]
+
+# Repository string must be defined in the publish register.py. See publish documentation for info.
+repo_map: {
+    'src/jvm/io/fsq': 'oss_sonatype_repo',
+    'src/thrift/io/fsq': 'oss_sonatype_repo',
+  }
+
+publication_metadata_map: {
+    'src/jvm/io/fsq': 'io_fsq_library("Fsq.io common libraries, from Foursquare.")',
+    'src/thrift/io/fsq': 'io_fsq_library("Fsq.io common libraries, from Foursquare.")'
+  }
+
+
 [tag]
 # Currently overridden internally, possible bug in dict_option.
 by_prefix:  +{
@@ -452,26 +483,23 @@ global_excludes: [
     {"org": "com.esotericsoftware", "name": "reflectasm"},
     {"org": "org.apache.hadoop", "name": "hadoop-client"},
     {"org": "org.apache.hadoop", "name": "hadoop-core"},
-    {"org": "io.netty", "name": "netty-buffer"},
-    {"org": "io.netty", "name": "netty-codec"},
-    {"org": "io.netty", "name": "netty-codec-dns"},
-    {"org": "io.netty", "name": "netty-codec-haproxy"},
-    {"org": "io.netty", "name": "netty-codec-http"},
-    {"org": "io.netty", "name": "netty-codec-http2"},
-    {"org": "io.netty", "name": "netty-codec-memcache"},
-    {"org": "io.netty", "name": "netty-codec-mqtt"},
-    {"org": "io.netty", "name": "netty-codec-smtp"},
-    {"org": "io.netty", "name": "netty-codec-socks"},
-    {"org": "io.netty", "name": "netty-codec-stomp"},
-    {"org": "io.netty", "name": "netty-common"},
-    {"org": "io.netty", "name": "netty-handler"},
-    {"org": "io.netty", "name": "netty-handler-proxy"},
-    {"org": "io.netty", "name": "netty-resolver"},
-    {"org": "io.netty", "name": "netty-resolver-dns"},
-    {"org": "io.netty", "name": "netty-transport"},
-    {"org": "io.netty", "name": "netty-transport-rxtx"},
-    {"org": "io.netty", "name": "netty-transport-sctp"},
-    {"org": "io.netty", "name": "netty-transport-ut"},
+    # NOTE(jacob): Depending on the monolithic netty jar rather than its individual
+    #   modules causes all kinds of headaches with other libraries that try to do the
+    #   right thing and only pull in what they need. There is some talk in the netty
+    #   project of making the monolith an empty wrapper that pulls in all the module jars
+    #   (see https://github.com/netty/netty/issues/4671 for details), but for now it
+    #   bundles duplicate classes and we have to sort out the conflict. Our dep report
+    #   lists spark as the only problem child pulling this in, but just one is enough.
+    #
+    #   What we do here is exclude the monolith, and instead declare a jar_library
+    #   containing the individual netty modules we need. If you add a new 3rdparty jar
+    #   with a dependency on netty-all, you may see classloading issues if it uses code
+    #   from a module we aren't pulling in. It is up to you to identify the missing
+    #   module, add it to our netty4 jar_library, and then add a dependency from your
+    #   new jar_library to our netty4 jar_library.
+    #
+    #   Note also included on the netty jar_library in 3rdparty/BUILD.opensource.
+    {"org": "io.netty", "name": "netty-all"},
     {"org": "org.jruby", "name": "jruby-complete"},
     # Twitter publishes modules with these direct dependencies - but they are internal only.
     {"org": "com.twitter", "name": "finagle-core-java"},