Merge pull request #69 from deel-ai/develop

Merge "develop" into "master" for release 1.4.0

.github/workflows/python-linters.yml

@@ -1,19 +1,24 @@
 name: deel-lip linters
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
 
 jobs:
   checks:
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.7, "3.10"]
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.github/workflows/python-publish.yml

@@ -13,9 +13,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.x'
     - name: Install dependencies

.github/workflows/python-tests.yml

@@ -1,24 +1,37 @@
 name: deel-lip tests
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+  schedule:
+    - cron: "0 2 * * 0" # Run tests every Sunday at 2am
 
 jobs:
   checks:
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        include:
+          - python-version: 3.7
+            tf-version: 2.3
+          - python-version: 3.9
+            tf-version: 2.7
+          - python-version: "3.10"
+            tf-version: latest
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install tox
-    - name: Test with tox
-      run: tox -e py$(echo ${{ matrix.python-version }} | tr -d .)
+    - name: Test with tox (Python ${{ matrix.python-version }} - TensorFlow ${{ matrix.tf-version }})
+      run: tox -e py$(echo ${{ matrix.python-version }}-tf${{ matrix.tf-version }} | tr -d .)

CONTRIBUTING.md

@@ -27,18 +27,12 @@ Before opening a pull request, please make sure you check your code and you run
 unit tests:
 
 ```bash
-# Using make:
 $ make test
-
-# Or using directly tox in your development environment:
-$ tox
 ```
 
 This command will:
 - check your code with black PEP-8 formatter and flake8 linter.
-- run `unittest` on the `tests/` folder with Python 3.6, 3.7 and 3.8.
-> Note: If you do not have those 3 interpreters, the tests will only be only performed
-  with your current interpreter.
+- run `unittest` on the `tests/` folder with different Python and TensorFlow versions.
 
 
 ## Submitting your changes

MANIFEST.in

@@ -0,0 +1,2 @@
+include deel/lip/VERSION
+include LICENSE

Makefile

@@ -22,10 +22,16 @@ prepare-dev:
 	. deel_lip_dev_env/bin/activate && pip install -e .[docs]
 
 test:
-	. deel_lip_dev_env/bin/activate && tox
+	. deel_lip_dev_env/bin/activate && tox -e py37-tf23
+	. deel_lip_dev_env/bin/activate && tox -e py39-tf27
+	. deel_lip_dev_env/bin/activate && tox -e py310-tflatest
+	. deel_lip_dev_env/bin/activate && tox -e py310-lint
 
 test-disable-gpu:
-	. deel_lip_dev_env/bin/activate && CUDA_VISIBLE_DEVICES=-1 tox
+	. deel_lip_dev_env/bin/activate && CUDA_VISIBLE_DEVICES=-1 tox -e py37-tf23
+	. deel_lip_dev_env/bin/activate && CUDA_VISIBLE_DEVICES=-1 tox -e py39-tf27
+	. deel_lip_dev_env/bin/activate && CUDA_VISIBLE_DEVICES=-1 tox -e py310-tflatest
+	. deel_lip_dev_env/bin/activate && CUDA_VISIBLE_DEVICES=-1 tox -e py310-lint
 
 doc:
 	. deel_lip_dev_env/bin/activate && cd doc && make html && cd -

deel/lip/VERSION

@@ -0,0 +1 @@
+1.4.0

deel/lip/__init__.py

@@ -2,6 +2,12 @@
 # rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
 # CRIAQ and ANITI - https://www.deel.ai/
 # =====================================================================================
+
+from os import path
+
+with open(path.join(path.dirname(__file__), "VERSION")) as f:
+    __version__ = f.read().strip()
+
 from . import activations
 from . import callbacks
 from . import constraints
@@ -12,4 +18,5 @@
 from . import metrics
 from .model import Sequential, Model, vanillaModel
 from . import normalizers
+from . import regularizers
 from . import utils

deel/lip/activations.py

@@ -7,6 +7,7 @@
 It can be added as a layer, or it can be used in the "activation" params for other
 layers.
 """
+import math
 import tensorflow as tf
 from tensorflow.keras import backend as K
 from tensorflow.keras.constraints import MinMaxNorm
@@ -211,3 +212,85 @@ def PReLUlip(k_coef_lip=1.0):
     return PReLU(
         alpha_constraint=MinMaxNorm(min_value=-k_coef_lip, max_value=k_coef_lip)
     )
+
+
+@register_keras_serializable("deel-lip", "Householder")
+class Householder(Layer, LipschitzLayer):
+    def __init__(
+        self,
+        data_format="channels_last",
+        k_coef_lip=1.0,
+        theta_initializer=None,
+        **kwargs,
+    ):
+        """
+        Householder activation: https://openreview.net/pdf?id=tD7eCtaSkR
+        From https://github.com/singlasahil14/SOC
+
+        Args:
+            data_format: either channels_first or channels_last. Only channels_last is
+                supported.
+            k_coef_lip: The lipschitz coefficient to be enforced.
+            theta_initializer: initializer for the angle theta of reflection. Defaults
+                to pi/2, which corresponds to GroupSort2.
+            **kwargs: parameters passed to the `tf.keras.layers.Layer`.
+
+        Input shape:
+            Arbitrary. Use the keyword argument `input_shape` (tuple of integers, does
+            not include the samples axis) when using this layer as the first layer in a
+            model.
+
+        Output shape:
+            Same size as input.
+
+        """
+        if data_format != "channels_last":
+            raise RuntimeError("Only 'channels_last' data format is supported")
+
+        self.data_format = data_format
+        self.set_klip_factor(k_coef_lip)
+        self.theta_initializer = theta_initializer
+        super().__init__(**kwargs)
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        self._init_lip_coef(input_shape)
+        if (input_shape[-1] % 2) != 0:
+            raise RuntimeError("2 has to be a divisor of the number of channels")
+
+        self.theta = self.add_weight(
+            "theta",
+            shape=[input_shape[-1] // 2],
+            initializer=self.theta_initializer,
+        )
+        if self.theta_initializer is None:
+            self.theta.assign(tf.ones_like(self.theta, dtype=tf.float32) * math.pi / 2)
+
+    def _compute_lip_coef(self, input_shape=None):
+        return 1.0
+
+    def call(self, x):
+        z1, z2 = tf.split(x, 2, axis=-1)
+
+        # selector > 0 if point (z1, z2) is on one side of reflection line, else < 0.
+        # Reflection line is defined by angle theta/2.
+        selector = (z1 * tf.sin(0.5 * self.theta)) - (z2 * tf.cos(0.5 * self.theta))
+
+        cos_theta = tf.cos(self.theta)
+        sin_theta = tf.sin(self.theta)
+        reflected_z1 = z1 * cos_theta + z2 * sin_theta
+        reflected_z2 = z1 * sin_theta - z2 * cos_theta
+
+        a = tf.where(selector <= 0, z1, reflected_z1)
+        b = tf.where(selector <= 0, z2, reflected_z2)
+
+        return tf.concat([a, b], axis=-1)
+
+    def get_config(self):
+        config = {
+            "k_coef_lip": self.k_coef_lip,
+            "data_format": self.data_format,
+            "theta_initializer": self.theta_initializer,
+        }
+        base_config = super().get_config()
+        return dict(list(base_config.items()) + list(config.items()))

deel/lip/callbacks.py

@@ -10,7 +10,7 @@
 
 import tensorflow as tf
 from tensorflow.keras.callbacks import Callback
-
+import numpy as np
 from .layers import Condensable
 
 
@@ -164,3 +164,65 @@ def get_config(self):
         }
         base_config = super(MonitorCallback, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
+
+
+class LossParamScheduler(Callback):
+    def __init__(self, param_name, fp, xp, step=0):
+        """
+        Scheduler to modify a loss parameter during training. It uses a linear
+        interpolation (defined by fp and xp) depending on the optimization step.
+
+        Args:
+            param_name (str): name of the parameter of the loss to tune. Must be a
+                tf.Variable.
+            fp (list): values of the loss parameter as steps given by the xp.
+            xp (list): step where the parameter equals fp.
+            step: step value, for serialization/deserialization purposes.
+        """
+        self.xp = xp
+        self.fp = fp
+        self.step = step
+        self.param_name = param_name
+
+    def on_train_batch_begin(self, batch: int, logs=None):
+        new_value = np.interp(self.step, self.xp, self.fp)
+        self.model.loss.__getattribute__(self.param_name).assign(new_value)
+        self.step += 1
+        super(LossParamScheduler, self).on_train_batch_end(batch, logs)
+
+    def get_config(self):
+        return {
+            "xp": self.xp,
+            "fp": self.fp,
+            "step": self.step,
+            "param_name": self.param_name,
+        }
+
+
+class LossParamLog(Callback):
+    def __init__(self, param_name, rate=1):
+        """
+        Logger to print values of a loss parameter at each epoch.
+
+        Args:
+            param_name (str): name of the parameter of the loss to log.
+            rate (int): logging rate (in epochs)
+        """
+        self.param_name = param_name
+        self.rate = rate
+
+    def on_epoch_end(self, epoch: int, logs=None):
+        if epoch % self.rate == 0:
+            tf.print(
+                "\n",
+                self.model.loss.name,
+                self.param_name,
+                self.model.loss.__getattribute__(self.param_name),
+            )
+        super(LossParamLog, self).on_train_batch_end(epoch, logs)
+
+    def get_config(self):
+        return {
+            "param_name": self.param_name,
+            "rate": self.rate,
+        }

deel/lip/initializers.py

@@ -2,7 +2,7 @@
 # rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
 # CRIAQ and ANITI - https://www.deel.ai/
 # =====================================================================================
-from tensorflow.keras.initializers import Initializer, Orthogonal
+from tensorflow.keras.initializers import Initializer
 from tensorflow.keras import initializers
 from .normalizers import (
     reshaped_kernel_orthogonalization,
@@ -21,7 +21,7 @@ def __init__(
         eps_bjorck=DEFAULT_EPS_BJORCK,
         beta_bjorck=DEFAULT_BETA_BJORCK,
         k_coef_lip=1.0,
-        base_initializer=Orthogonal(gain=1.0, seed=None),
+        base_initializer="orthogonal",
     ) -> None:
         """
         Initialize a kernel to be 1-lipschitz orthogonal using bjorck