Vectors (#9)

* Add Entities

* turn off rules

* Rename to vectors and improve cli interface

* fix vector writes

* nest dimensions

* update readme

* add usage

* snip extra docs

* more bdd

* logger spy

* Generate rubocop todo

* split tests

* load yaml

* more yaml parsing

* clean up cops

* rm empty todo

* snip

* squash test log output

* absolute

* more absolute

* more absolute

* more absolute

* improve test legibility

* Respect example length rule

* Respect nesting rule

* enforce docs

* absolute

* dry

* api extraction

* vector extraction

* remove redundant FakeFS

* cleaner

* more assertion singularization

* snip

* further API extrapolation

* Add spec persistence

* pass tests again

* snip

* 3.0 is past EOL

* fix coverage

* refactor CLI spec

* remove unnecessary context

* more review feedback

* more simplification

* more reduction

* safe load

* one more

* no need to manually activate / deactivate fakefs

* Add more coverage to with-directory project specs

* force pathname

* more pathname

* rename to fields

* a million pathnames

* use pathname methods

* path not a file

* improve test log

* Implement Project.create as a class method

* shorter line

* pathname .freeze

* turn back on

* improve memoized helpers

* Pass rubocops

* delete unused

* vectors do not create routines or tables directories

* no todos

* tighten

* tighten

* snip

* abstract ugly template setup

* nicen

* read and write

* more comments

* bump version

---------

Co-authored-by: tyler <tyler@tyler.love>

.github/workflows/ci.yml

@@ -14,10 +14,9 @@ jobs:
     strategy:
       matrix:
         ruby:
-          - '3.0.6'
-          - '3.1.4'
-          - '3.2.2'
-          - '3.3.5'
+          - '3.1.6'
+          - '3.2.6'
+          - '3.3.6'
 
     steps:
       - uses: actions/checkout@v4

.gitignore

@@ -5,6 +5,7 @@
 /doc/
 /pkg/
 /spec/reports/
+/spec/examples.txt
 /tmp/
 *.gem
 

.rubocop.yml

@@ -1,3 +1,5 @@
+inherit_from: .rubocop_todo.yml
+
 require: rubocop-rspec
 
 AllCops:

.rubocop_todo.yml

@@ -0,0 +1,7 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2024-11-09 13:21:19 UTC using RuboCop version 1.68.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.

Gemfile

@@ -11,6 +11,8 @@ group :development do
 end
 
 group :development, :test do
+  gem "debug"
+  gem "fakefs"
   gem "rspec"
   gem "simplecov"
   gem "simplecov-json"

Gemfile.lock

@@ -2,18 +2,32 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
+    debug (1.9.2)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
     diff-lcs (1.5.1)
     docile (1.4.1)
+    fakefs (2.5.0)
+    io-console (0.7.2)
+    irb (1.14.1)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
     json (2.7.2)
     language_server-protocol (3.17.0.3)
     logger (1.6.1)
     parallel (1.24.0)
     parser (3.3.1.0)
       ast (~> 2.4.1)
       racc
+    psych (5.1.2)
+      stringio
     racc (1.7.3)
     rainbow (3.1.1)
+    rdoc (6.7.0)
+      psych (>= 4.0.0)
     regexp_parser (2.9.0)
+    reline (0.5.10)
+      io-console (~> 0.5)
     rexml (3.2.6)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -63,6 +77,7 @@ GEM
       simplecov
     simplecov-lcov (0.8.0)
     simplecov_json_formatter (0.1.4)
+    stringio (3.1.1)
     thor (1.3.1)
     unicode-display_width (2.5.0)
 
@@ -71,6 +86,8 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  debug
+  fakefs
   logger
   rspec
   rubocop

README.md

@@ -66,16 +66,19 @@ manifolds generate <data_project_name> bq
 
 ## Manifolds Configuration
 
-### Dimensions
+### Vectors
 
-Dimensions are fields that describe the context of the data. They are typically used to segment and filter data in reports.
+Vectors are the entities you can roll up data for. Each vector has a set of dimensions defined in its `vectors/<vector_name>.yml` configuration file.
 
 ```yaml
-dimensions:
-  - name: user_id
-    type: STRING
-  - name: date
-    type: DATE
+vectors:
+  - page
+```
+
+#### Add a vector to your project
+
+```bash
+manifolds vectors add page
 ```
 
 ### Metrics

lib/manifolds.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
-require_relative "manifolds/cli"
-require_relative "manifolds/version"
+require "pathname"
+require "thor"
+require "yaml"
+
+Dir[File.join(__dir__, "manifolds", "**", "*.rb")].sort.each do |file|
+  require file
+end
 
 module Manifolds
   class Error < StandardError; end

lib/manifolds/api.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Manifolds
+  # API for interacting with project folders and file structures.
+  module API
+  end
+end

lib/manifolds/cli.rb

@@ -1,59 +1,60 @@
 # frozen_string_literal: true
 
-require "thor"
-require "fileutils"
-require "logger"
-
-require_relative "services/big_query_service"
-
 module Manifolds
   # CLI provides command line interface functionality
   # for creating and managing umbrella projects for data management.
   class CLI < Thor
+    attr_accessor :logger, :bq_service
+
     def initialize(*args, logger: Logger.new($stdout))
       super(*args)
-      @logger = logger
-      @logger.level = Logger::INFO
 
-      @bq_service = Services::BigQueryService.new(@logger)
+      self.logger = logger
+      logger.level = Logger::INFO
+
+      self.bq_service = Services::BigQueryService.new(logger)
     end
 
     desc "init NAME", "Generate a new umbrella project for data management"
     def init(name)
-      directory_path = "./#{name}/projects"
-      FileUtils.mkdir_p(directory_path)
-      @logger.info "Created umbrella project '#{name}' with a projects directory."
+      Manifolds::API::Project.create(name)
+      logger.info "Created umbrella project '#{name}' with projects and vectors directories."
     end
 
-    desc "add PROJECT_NAME", "Add a new project within the current umbrella project"
-    def add(project_name)
-      project_path = "./projects/#{project_name}"
-      unless Dir.exist?("./projects")
-        @logger.error("Not inside a Manifolds umbrella project.")
-        return
+    desc "vectors SUBCOMMAND ...ARGS", "Manage vectors"
+    subcommand "vectors", Class.new(Thor) {
+      namespace :vectors
+
+      attr_accessor :logger
+
+      def initialize(*args, logger: Logger.new($stdout))
+        super(*args)
+        self.logger = logger
       end
 
-      FileUtils.mkdir_p("#{project_path}/tables")
-      FileUtils.mkdir_p("#{project_path}/routines")
-      copy_config_template(project_path)
-      @logger.info "Added project '#{project_name}' with tables and routines directories."
+      desc "add VECTOR_NAME", "Add a new vector configuration"
+      def add(name, project: API::Project.new(File.basename(Dir.getwd)))
+        vector = API::Vector.new(name, project: project)
+        vector.add
+        logger.info "Created vector configuration for '#{name}'."
+      end
+    }
+
+    desc "add WORKSPACE_NAME", "Add a new workspace to a project"
+    def add(name, project: API::Project.new(File.basename(Dir.getwd)))
+      workspace = API::Workspace.new(name, project: project)
+      workspace.add
+      logger.info "Added workspace '#{name}' with tables and routines directories."
     end
 
     desc "generate PROJECT_NAME SERVICE", "Generate services for a project"
     def generate(project_name, service)
       case service
       when "bq"
-        @bq_service.generate_dimensions_schema(project_name)
+        bq_service.generate_dimensions_schema(project_name)
       else
-        @logger.error("Unsupported service: #{service}")
+        logger.error("Unsupported service: #{service}")
       end
     end
-
-    private
-
-    def copy_config_template(project_path)
-      template_path = File.join(File.dirname(__FILE__), "templates", "config_template.yml")
-      FileUtils.cp(template_path, "#{project_path}/manifold.yml")
-    end
   end
 end

lib/manifolds/project/project.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Manifolds
+  module API
+    # Projects API
+    class Project
+      attr_reader :name, :directory
+
+      def initialize(name, directory: Pathname.pwd.join(name))
+        self.name = name
+        self.directory = Pathname(directory)
+      end
+
+      def self.create(name, directory: Pathname.pwd.join(name))
+        new(name, directory: directory).tap do |project|
+          [project.workspaces_directory, project.vectors_directory].each(&:mkpath)
+        end
+      end
+
+      def workspaces_directory
+        directory.join("workspaces")
+      end
+
+      def vectors_directory
+        directory.join("vectors")
+      end
+
+      private
+
+      attr_writer :name, :directory
+    end
+  end
+end

lib/manifolds/project/vector.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Manifolds
+  module API
+    # Describes the entities for whom metrics are calculated.
+    class Vector
+      attr_reader :name, :project, :template_path
+
+      DEFAULT_TEMPLATE_PATH = Pathname.pwd.join(
+        "lib", "manifolds", "templates", "vector_template.yml"
+      ).freeze
+
+      def initialize(name, project:, template_path: DEFAULT_TEMPLATE_PATH)
+        self.name = name
+        self.project = project
+        self.template_path = Pathname(template_path)
+      end
+
+      def add
+        directory.mkpath
+        FileUtils.cp(template_path, config_path)
+      end
+
+      private
+
+      attr_writer :name, :project, :template_path
+
+      def directory
+        project.directory.join("vectors")
+      end
+
+      def config_path
+        directory.join("#{name.downcase}.yml")
+      end
+    end
+  end
+end

lib/manifolds/project/workspace.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Manifolds
+  module API
+    # Encapsulates a single manifold.
+    class Workspace
+      attr_reader :name, :project, :template_path
+
+      DEFAULT_TEMPLATE_PATH = Pathname.pwd.join(
+        "lib", "manifolds", "templates", "workspace_template.yml"
+      )
+
+      def initialize(name, project:, template_path: DEFAULT_TEMPLATE_PATH)
+        self.name = name
+        self.project = project
+        self.template_path = template_path
+      end
+
+      def add
+        [tables_directory, routines_directory].each(&:mkpath)
+        FileUtils.cp(template_path, manifold_path)
+      end
+
+      def tables_directory
+        project.workspaces_directory.join(name, "tables")
+      end
+
+      def routines_directory
+        project.workspaces_directory.join(name, "routines")
+      end
+
+      def manifold_file
+        return nil unless manifold_exists?
+
+        File.new(manifold_path)
+      end
+
+      def manifold_exists?
+        manifold_path.file?
+      end
+
+      def manifold_path
+        project.workspaces_directory.join(name, "manifold.yml")
+      end
+
+      private
+
+      attr_writer :name, :project, :template_path
+    end
+  end
+end