Add performance benchmarking scripts (#1979)

* benchmarks: Upload initial models from JuliaDynamics

https://github.com/JuliaDynamics/ABMFrameworksComparison/tree/5551d7abf1611d377b3b32346c7774f176af4c65

* benchmarks: Add dictionary with configurations

* benchmarks: Update configurations, use relative imports

* benchmarks: Add single script to run all benchmarks

* benchmarks: Update configurations

Few less replications, some more seeds. Every benchmark now takes between 10 and 20 seconds (on my machine).

* benchmarks: Add generated pickle files to gitignore

That allows switching branches without benchmarks results disappearing

* benchmarks: Update global script to calculate and save stuff

Prints some stuff when running and saves a pickle file after running.

* benchmarks: Write a script to statistically compare runs

- The bootstrap_speedup_confidence_interval function calculates the mean speedup and its confidence interval using bootstrapping, which is more suitable for paired data.
- The mean speedup and confidence interval are calculated for both initialization and run times.
- Positive values indicate an increase in time (longer duration), and negative values indicate a decrease (shorter duration).
- The results are displayed in a DataFrame with the percentage changes and their confidence intervals.

* benchmarks: Remove seperate benchmark scripts

* benchmarks: Black and ruff

* benchmarks: Use old f-string formatting to work with Python 3.11 and older

* benchmarks: Add fancy colored performance indicators 🟢🔴🔵

If the 95% confidence interval is:
- fully below -3%: 🟢
- fully above +3%: 🔴
- else: 🔵

* Fix typos caught by codespell

* Apply ruff format benchmarks/

* Apply ruff --fix benchmarks/

* Apply ruff --fix --unsafe-fixes benchmarks/

* Apply manual Ruff fixes

* codecov: Ignore benchmarks/

---------

Co-authored-by: rht <rhtbot@protonmail.com>

.gitignore

@@ -1,3 +1,6 @@
+# Benchmarking
+benchmarking/**/*.pickle
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

benchmarks/Flocking/Flocking.py

@@ -0,0 +1,80 @@
+"""
+Flockers
+=============================================================
+A Mesa implementation of Craig Reynolds's Boids flocker model.
+Uses numpy arrays to represent vectors.
+"""
+
+import numpy as np
+
+from mesa import Model
+from mesa.space import ContinuousSpace
+from mesa.time import RandomActivation
+
+from .boid import Boid
+
+
+class BoidFlockers(Model):
+    """
+    Flocker model class. Handles agent creation, placement and scheduling.
+    """
+
+    def __init__(
+        self,
+        seed,
+        population,
+        width,
+        height,
+        vision,
+        speed=1,
+        separation=1,
+        cohere=0.03,
+        separate=0.015,
+        match=0.05,
+    ):
+        """
+        Create a new Flockers model.
+
+        Args:
+            population: Number of Boids
+            width, height: Size of the space.
+            speed: How fast should the Boids move.
+            vision: How far around should each Boid look for its neighbors
+            separation: What's the minimum distance each Boid will attempt to
+                    keep from any other
+            cohere, separate, match: factors for the relative importance of
+                    the three drives."""
+        super().__init__(seed=seed)
+        self.population = population
+        self.vision = vision
+        self.speed = speed
+        self.separation = separation
+        self.schedule = RandomActivation(self)
+        self.space = ContinuousSpace(width, height, True)
+        self.factors = {"cohere": cohere, "separate": separate, "match": match}
+        self.make_agents()
+
+    def make_agents(self):
+        """
+        Create self.population agents, with random positions and starting headings.
+        """
+        for i in range(self.population):
+            x = self.random.random() * self.space.x_max
+            y = self.random.random() * self.space.y_max
+            pos = np.array((x, y))
+            velocity = np.random.random(2) * 2 - 1
+            boid = Boid(
+                i,
+                self,
+                pos,
+                self.speed,
+                velocity,
+                self.vision,
+                self.separation,
+                **self.factors,
+            )
+            self.space.place_agent(boid, pos)
+            self.schedule.add(boid)
+
+    def step(self):
+        self.schedule.step()

benchmarks/Flocking/boid.py

@@ -0,0 +1,81 @@
+import numpy as np
+
+from mesa import Agent
+
+
+class Boid(Agent):
+    """
+    A Boid-style flocker agent.
+
+    The agent follows three behaviors to flock:
+        - Cohesion: steering towards neighboring agents.
+        - Separation: avoiding getting too close to any other agent.
+        - Alignment: try to fly in the same direction as the neighbors.
+
+    Boids have a vision that defines the radius in which they look for their
+    neighbors to flock with. Their speed (a scalar) and velocity (a vector)
+    define their movement. Separation is their desired minimum distance from
+    any other Boid.
+    """
+
+    def __init__(
+        self,
+        unique_id,
+        model,
+        pos,
+        speed,
+        velocity,
+        vision,
+        separation,
+        cohere=0.03,
+        separate=0.015,
+        match=0.05,
+    ):
+        """
+        Create a new Boid flocker agent.
+
+        Args:
+            unique_id: Unique agent identifier.
+            pos: Starting position
+            speed: Distance to move per step.
+            heading: numpy vector for the Boid's direction of movement.
+            vision: Radius to look around for nearby Boids.
+            separation: Minimum distance to maintain from other Boids.
+            cohere: the relative importance of matching neighbors' positions
+            separate: the relative importance of avoiding close neighbors
+            match: the relative importance of matching neighbors' headings
+
+        """
+        super().__init__(unique_id, model)
+        self.pos = np.array(pos)
+        self.speed = speed
+        self.velocity = velocity
+        self.vision = vision
+        self.separation = separation
+        self.cohere_factor = cohere
+        self.separate_factor = separate
+        self.match_factor = match
+
+    def step(self):
+        """
+        Get the Boid's neighbors, compute the new vector, and move accordingly.
+        """
+
+        neighbors = self.model.space.get_neighbors(self.pos, self.vision, False)
+        n = 0
+        match_vector, separation_vector, cohere = np.zeros((3, 2))
+        for neighbor in neighbors:
+            n += 1
+            heading = self.model.space.get_heading(self.pos, neighbor.pos)
+            cohere += heading
+            if self.model.space.get_distance(self.pos, neighbor.pos) < self.separation:
+                separation_vector -= heading
+            match_vector += neighbor.velocity
+        n = max(n, 1)
+        cohere = cohere * self.cohere_factor
+        separation_vector = separation_vector * self.separate_factor
+        match_vector = match_vector * self.match_factor
+        self.velocity += (cohere + separation_vector + match_vector) / n
+        self.velocity /= np.linalg.norm(self.velocity)
+        new_pos = self.pos + self.velocity * self.speed
+        self.model.space.move_agent(self, new_pos)

benchmarks/Schelling/Schelling.py

@@ -0,0 +1,77 @@
+import random
+
+from mesa import Agent, Model
+from mesa.space import SingleGrid
+from mesa.time import RandomActivation
+
+
+class SchellingAgent(Agent):
+    """
+    Schelling segregation agent
+    """
+
+    def __init__(self, pos, model, agent_type):
+        """
+        Create a new Schelling agent.
+        Args:
+           unique_id: Unique identifier for the agent.
+           x, y: Agent initial location.
+           agent_type: Indicator for the agent's type (minority=1, majority=0)
+        """
+        super().__init__(pos, model)
+        self.pos = pos
+        self.type = agent_type
+
+    def step(self):
+        similar = 0
+        r = self.model.radius
+        for neighbor in self.model.grid.iter_neighbors(self.pos, moore=True, radius=r):
+            if neighbor.type == self.type:
+                similar += 1
+
+        # If unhappy, move:
+        if similar < self.model.homophily:
+            self.model.grid.move_to_empty(self)
+        else:
+            self.model.happy += 1
+
+
+class SchellingModel(Model):
+    """
+    Model class for the Schelling segregation model.
+    """
+
+    def __init__(
+        self, seed, height, width, homophily, radius, density, minority_pc=0.5
+    ):
+        """ """
+        super().__init__(seed=seed)
+        self.height = height
+        self.width = width
+        self.density = density
+        self.minority_pc = minority_pc
+        self.homophily = homophily
+        self.radius = radius
+
+        self.schedule = RandomActivation(self)
+        self.grid = SingleGrid(height, width, torus=True)
+
+        self.happy = 0
+
+        # Set up agents
+        # We use a grid iterator that returns
+        # the coordinates of a cell as well as
+        # its contents. (coord_iter)
+        for _cont, pos in self.grid.coord_iter():
+            if random.random() < self.density:  # noqa: S311
+                agent_type = 1 if random.random() < self.minority_pc else 0  # noqa: S311
+                agent = SchellingAgent(pos, self, agent_type)
+                self.grid.place_agent(agent, pos)
+                self.schedule.add(agent)
+
+    def step(self):
+        """
+        Run one step of the model.
+        """
+        self.happy = 0  # Reset counter of happy agents
+        self.schedule.step()

benchmarks/WolfSheep/WolfSheep.py

@@ -0,0 +1,103 @@
+"""
+Wolf-Sheep Predation Model
+================================
+
+Replication of the model found in NetLogo:
+    Wilensky, U. (1997). NetLogo Wolf Sheep Predation model.
+    http://ccl.northwestern.edu/netlogo/models/WolfSheepPredation.
+    Center for Connected Learning and Computer-Based Modeling,
+    Northwestern University, Evanston, IL.
+"""
+
+import mesa
+from mesa.space import MultiGrid
+from mesa.time import RandomActivationByType
+
+from .agents import GrassPatch, Sheep, Wolf
+
+
+class WolfSheep(mesa.Model):
+    """
+    Wolf-Sheep Predation Model
+
+    A model for simulating wolf and sheep (predator-prey) ecosystem modelling.
+    """
+
+    def __init__(
+        self,
+        seed,
+        height,
+        width,
+        initial_sheep,
+        initial_wolves,
+        sheep_reproduce,
+        wolf_reproduce,
+        grass_regrowth_time,
+        wolf_gain_from_food=13,
+        sheep_gain_from_food=5,
+    ):
+        """
+        Create a new Wolf-Sheep model with the given parameters.
+
+        Args:
+            initial_sheep: Number of sheep to start with
+            initial_wolves: Number of wolves to start with
+            sheep_reproduce: Probability of each sheep reproducing each step
+            wolf_reproduce: Probability of each wolf reproducing each step
+            wolf_gain_from_food: Energy a wolf gains from eating a sheep
+            grass: Whether to have the sheep eat grass for energy
+            grass_regrowth_time: How long it takes for a grass patch to regrow
+                                 once it is eaten
+            sheep_gain_from_food: Energy sheep gain from grass, if enabled.
+        """
+        super().__init__(seed=seed)
+        # Set parameters
+        self.height = height
+        self.width = width
+        self.initial_sheep = initial_sheep
+        self.initial_wolves = initial_wolves
+        self.sheep_reproduce = sheep_reproduce
+        self.wolf_reproduce = wolf_reproduce
+        self.wolf_gain_from_food = wolf_gain_from_food
+        self.grass_regrowth_time = grass_regrowth_time
+        self.sheep_gain_from_food = sheep_gain_from_food
+
+        self.schedule = RandomActivationByType(self)
+        self.grid = MultiGrid(self.height, self.width, torus=False)
+
+        # Create sheep:
+        for _i in range(self.initial_sheep):
+            pos = (
+                self.random.randrange(self.width),
+                self.random.randrange(self.height),
+            )
+            energy = self.random.randrange(2 * self.sheep_gain_from_food)
+            sheep = Sheep(self.next_id(), pos, self, True, energy)
+            self.grid.place_agent(sheep, pos)
+            self.schedule.add(sheep)
+
+        # Create wolves
+        for _i in range(self.initial_wolves):
+            pos = (
+                self.random.randrange(self.width),
+                self.random.randrange(self.height),
+            )
+            energy = self.random.randrange(2 * self.wolf_gain_from_food)
+            wolf = Wolf(self.next_id(), pos, self, True, energy)
+            self.grid.place_agent(wolf, pos)
+            self.schedule.add(wolf)
+
+        # Create grass patches
+        possibly_fully_grown = [True, False]
+        for _agent, pos in self.grid.coord_iter():
+            fully_grown = self.random.choice(possibly_fully_grown)
+            if fully_grown:
+                countdown = self.grass_regrowth_time
+            else:
+                countdown = self.random.randrange(self.grass_regrowth_time)
+            patch = GrassPatch(self.next_id(), pos, self, fully_grown, countdown)
+            self.grid.place_agent(patch, pos)
+            self.schedule.add(patch)
+
+    def step(self):
+        self.schedule.step()