colaborative_filtering_spotlight.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"1-s5qLpfzPJ_UIGGt0cfbEDUj3GmJie0H","timestamp":1710213495585}],"machine_shape":"hm","gpuType":"T4","collapsed_sections":["rAInlcpYqX-A"]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["## Download the dataset - from Kaggle\n","\n","Skip this step if you have the data in Drive already!"],"metadata":{"id":"rAInlcpYqX-A"}},{"cell_type":"code","execution_count":5,"metadata":{"id":"h6MgHB3BLIAP","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1710218104529,"user_tz":420,"elapsed":5503,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"dd574d82-fcec-4ef6-fd60-9fc2b8c9da89"},"outputs":[{"output_type":"stream","name":"stdout","text":["Traceback (most recent call last):\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3108, in _dep_map\n","    return self.__dep_map\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 2901, in __getattr__\n","    raise AttributeError(attr)\n","AttributeError: _DistInfoDistribution__dep_map\n","\n","During handling of the above exception, another exception occurred:\n","\n","Traceback (most recent call last):\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 169, in exc_logging_wrapper\n","    status = run_func(*args)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/req_command.py\", line 242, in wrapper\n","    return func(self, options, args)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/commands/install.py\", line 441, in run\n","    conflicts = self._determine_conflicts(to_install)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/commands/install.py\", line 572, in _determine_conflicts\n","    return check_install_conflicts(to_install)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/operations/check.py\", line 101, in check_install_conflicts\n","    package_set, _ = create_package_set_from_installed()\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/operations/check.py\", line 42, in create_package_set_from_installed\n","    dependencies = list(dist.iter_dependencies())\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/metadata/pkg_resources.py\", line 216, in iter_dependencies\n","    return self._dist.requires(extras)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 2821, in requires\n","    dm = self._dep_map\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3110, in _dep_map\n","    self.__dep_map = self._compute_dependencies()\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3120, in _compute_dependencies\n","    reqs.extend(parse_requirements(req))\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3173, in __init__\n","    super(Requirement, self).__init__(requirement_string)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/packaging/requirements.py\", line 102, in __init__\n","    req = REQUIREMENT.parseString(requirement_string)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 1131, in parse_string\n","    loc, tokens = self._parse(instring, 0)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n","    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 3886, in parseImpl\n","    loc, exprtokens = e._parse(instring, loc, doActions)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n","    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 4114, in parseImpl\n","    return e._parse(\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n","    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 3864, in parseImpl\n","    loc, resultlist = self.exprs[0]._parse(\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n","    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 3886, in parseImpl\n","    loc, exprtokens = e._parse(instring, loc, doActions)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n","    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 4959, in parseImpl\n","    loc, tokens = self_expr._parse(instring, loc, doActions, callPreParse=False)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n","    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 4114, in parseImpl\n","    return e._parse(\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 811, in _parseNoCache\n","    pre_loc = self.preParse(instring, loc)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_vendor/pyparsing/core.py\", line 764, in preParse\n","    while loc < instrlen and instring[loc] in white_chars:\n","KeyboardInterrupt\n","\n","During handling of the above exception, another exception occurred:\n","\n","Traceback (most recent call last):\n","  File \"/usr/local/bin/pip3\", line 8, in <module>\n","    sys.exit(main())\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/main.py\", line 79, in main\n","    return command.main(cmd_args)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 101, in main\n","    return self._main(args)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 223, in _main\n","    return run(options, args)\n","  File \"/usr/local/lib/python3.10/dist-packages/pip/_internal/cli/base_command.py\", line 206, in exc_logging_wrapper\n","    logger.critical(\"Operation cancelled by user\")\n","  File \"/usr/lib/python3.10/logging/__init__.py\", line 1524, in critical\n","    self._log(CRITICAL, msg, args, **kwargs)\n","  File \"/usr/lib/python3.10/logging/__init__.py\", line 1622, in _log\n","    record = self.makeRecord(self.name, level, fn, lno, msg, args,\n","  File \"/usr/lib/python3.10/logging/__init__.py\", line 1591, in makeRecord\n","    rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func,\n","  File \"/usr/lib/python3.10/logging/__init__.py\", line 328, in __init__\n","    self.msecs = int((ct - int(ct)) * 1000) + 0.0  # see gh-89047\n","KeyboardInterrupt\n","^C\n"]}],"source":["!pip install -q kaggle"]},{"cell_type":"code","source":["from google.colab import files\n","files.upload()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":350},"id":"dnoEwJAJLw3f","outputId":"1512fcb8-c92b-4bfb-90ce-6cc8bb11ef4a","executionInfo":{"status":"error","timestamp":1710218106269,"user_tz":420,"elapsed":1746,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":6,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","     <input type=\"file\" id=\"files-939198f1-9228-45a1-8996-d05b4ea4caae\" name=\"files[]\" multiple disabled\n","        style=\"border:none\" />\n","     <output id=\"result-939198f1-9228-45a1-8996-d05b4ea4caae\">\n","      Upload widget is only available when the cell has been executed in the\n","      current browser session. Please rerun this cell to enable.\n","      </output>\n","      <script>// Copyright 2017 Google LLC\n","//\n","// Licensed under the Apache License, Version 2.0 (the \"License\");\n","// you may not use this file except in compliance with the License.\n","// You may obtain a copy of the License at\n","//\n","//      http://www.apache.org/licenses/LICENSE-2.0\n","//\n","// Unless required by applicable law or agreed to in writing, software\n","// distributed under the License is distributed on an \"AS IS\" BASIS,\n","// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n","// See the License for the specific language governing permissions and\n","// limitations under the License.\n","\n","/**\n"," * @fileoverview Helpers for google.colab Python module.\n"," */\n","(function(scope) {\n","function span(text, styleAttributes = {}) {\n","  const element = document.createElement('span');\n","  element.textContent = text;\n","  for (const key of Object.keys(styleAttributes)) {\n","    element.style[key] = styleAttributes[key];\n","  }\n","  return element;\n","}\n","\n","// Max number of bytes which will be uploaded at a time.\n","const MAX_PAYLOAD_SIZE = 100 * 1024;\n","\n","function _uploadFiles(inputId, outputId) {\n","  const steps = uploadFilesStep(inputId, outputId);\n","  const outputElement = document.getElementById(outputId);\n","  // Cache steps on the outputElement to make it available for the next call\n","  // to uploadFilesContinue from Python.\n","  outputElement.steps = steps;\n","\n","  return _uploadFilesContinue(outputId);\n","}\n","\n","// This is roughly an async generator (not supported in the browser yet),\n","// where there are multiple asynchronous steps and the Python side is going\n","// to poll for completion of each step.\n","// This uses a Promise to block the python side on completion of each step,\n","// then passes the result of the previous step as the input to the next step.\n","function _uploadFilesContinue(outputId) {\n","  const outputElement = document.getElementById(outputId);\n","  const steps = outputElement.steps;\n","\n","  const next = steps.next(outputElement.lastPromiseValue);\n","  return Promise.resolve(next.value.promise).then((value) => {\n","    // Cache the last promise value to make it available to the next\n","    // step of the generator.\n","    outputElement.lastPromiseValue = value;\n","    return next.value.response;\n","  });\n","}\n","\n","/**\n"," * Generator function which is called between each async step of the upload\n"," * process.\n"," * @param {string} inputId Element ID of the input file picker element.\n"," * @param {string} outputId Element ID of the output display.\n"," * @return {!Iterable<!Object>} Iterable of next steps.\n"," */\n","function* uploadFilesStep(inputId, outputId) {\n","  const inputElement = document.getElementById(inputId);\n","  inputElement.disabled = false;\n","\n","  const outputElement = document.getElementById(outputId);\n","  outputElement.innerHTML = '';\n","\n","  const pickedPromise = new Promise((resolve) => {\n","    inputElement.addEventListener('change', (e) => {\n","      resolve(e.target.files);\n","    });\n","  });\n","\n","  const cancel = document.createElement('button');\n","  inputElement.parentElement.appendChild(cancel);\n","  cancel.textContent = 'Cancel upload';\n","  const cancelPromise = new Promise((resolve) => {\n","    cancel.onclick = () => {\n","      resolve(null);\n","    };\n","  });\n","\n","  // Wait for the user to pick the files.\n","  const files = yield {\n","    promise: Promise.race([pickedPromise, cancelPromise]),\n","    response: {\n","      action: 'starting',\n","    }\n","  };\n","\n","  cancel.remove();\n","\n","  // Disable the input element since further picks are not allowed.\n","  inputElement.disabled = true;\n","\n","  if (!files) {\n","    return {\n","      response: {\n","        action: 'complete',\n","      }\n","    };\n","  }\n","\n","  for (const file of files) {\n","    const li = document.createElement('li');\n","    li.append(span(file.name, {fontWeight: 'bold'}));\n","    li.append(span(\n","        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n","        `last modified: ${\n","            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n","                                    'n/a'} - `));\n","    const percent = span('0% done');\n","    li.appendChild(percent);\n","\n","    outputElement.appendChild(li);\n","\n","    const fileDataPromise = new Promise((resolve) => {\n","      const reader = new FileReader();\n","      reader.onload = (e) => {\n","        resolve(e.target.result);\n","      };\n","      reader.readAsArrayBuffer(file);\n","    });\n","    // Wait for the data to be ready.\n","    let fileData = yield {\n","      promise: fileDataPromise,\n","      response: {\n","        action: 'continue',\n","      }\n","    };\n","\n","    // Use a chunked sending to avoid message size limits. See b/62115660.\n","    let position = 0;\n","    do {\n","      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n","      const chunk = new Uint8Array(fileData, position, length);\n","      position += length;\n","\n","      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n","      yield {\n","        response: {\n","          action: 'append',\n","          file: file.name,\n","          data: base64,\n","        },\n","      };\n","\n","      let percentDone = fileData.byteLength === 0 ?\n","          100 :\n","          Math.round((position / fileData.byteLength) * 100);\n","      percent.textContent = `${percentDone}% done`;\n","\n","    } while (position < fileData.byteLength);\n","  }\n","\n","  // All done.\n","  yield {\n","    response: {\n","      action: 'complete',\n","    }\n","  };\n","}\n","\n","scope.google = scope.google || {};\n","scope.google.colab = scope.google.colab || {};\n","scope.google.colab._files = {\n","  _uploadFiles,\n","  _uploadFilesContinue,\n","};\n","})(self);\n","</script> "]},"metadata":{}},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-6-5c2e8a8d365b>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mfiles\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/files.py\u001b[0m in \u001b[0;36mupload\u001b[0;34m()\u001b[0m\n\u001b[1;32m     67\u001b[0m   \"\"\"\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m   \u001b[0muploaded_files\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_upload_files\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmultiple\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m   \u001b[0;31m# Mapping from original filename to filename as saved locally.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m   \u001b[0mlocal_filenames\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/files.py\u001b[0m in \u001b[0;36m_upload_files\u001b[0;34m(multiple)\u001b[0m\n\u001b[1;32m    154\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m   \u001b[0;31m# First result is always an indication that the file picker has completed.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m   result = _output.eval_js(\n\u001b[0m\u001b[1;32m    157\u001b[0m       'google.colab._files._uploadFiles(\"{input_id}\", \"{output_id}\")'.format(\n\u001b[1;32m    158\u001b[0m           \u001b[0minput_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_id\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/output/_js.py\u001b[0m in \u001b[0;36meval_js\u001b[0;34m(script, ignore_result, timeout_sec)\u001b[0m\n\u001b[1;32m     38\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mignore_result\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m     \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0m_message\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_reply_from_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout_sec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/_message.py\u001b[0m in \u001b[0;36mread_reply_from_input\u001b[0;34m(message_id, timeout_sec)\u001b[0m\n\u001b[1;32m     94\u001b[0m     \u001b[0mreply\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_read_next_input_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreply\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_NOT_READY\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreply\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m       \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.025\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m       \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m     if (\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"cell_type":"code","source":["! mkdir ~/.kaggle\n","! cp kaggle.json ~/.kaggle/"],"metadata":{"id":"zP4iQGkjL40A","executionInfo":{"status":"aborted","timestamp":1710218106270,"user_tz":420,"elapsed":5,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["! chmod 600 ~/.kaggle/kaggle.json"],"metadata":{"id":"l4hMxD53L8l7","executionInfo":{"status":"aborted","timestamp":1710218106270,"user_tz":420,"elapsed":5,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["! kaggle datasets list"],"metadata":{"id":"jn-8YohlL-Rw","executionInfo":{"status":"aborted","timestamp":1710218106270,"user_tz":420,"elapsed":5,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["! kaggle competitions download -c bgu-recommendation-systems-click-through-rate"],"metadata":{"id":"2QQfqzTCMKEX","executionInfo":{"status":"aborted","timestamp":1710218106270,"user_tz":420,"elapsed":5,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import zipfile\n","import os\n","\n","zip_file_path = 'bgu-recommendation-systems-click-through-rate.zip'\n","extracted_folder_path = 'bgu-recommendation-systems-click-through-rate'\n","\n","# Create the extraction directory if it doesn't exist\n","os.makedirs(extracted_folder_path, exist_ok=True)\n","\n","# Open the zip file and extract its contents\n","with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n","    zip_ref.extractall(extracted_folder_path)\n","\n","print(f\"Successfully extracted contents to {extracted_folder_path}\")"],"metadata":{"id":"cZytyofcOdBc","executionInfo":{"status":"aborted","timestamp":1710218106270,"user_tz":420,"elapsed":5,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Download the dataset - from Drive"],"metadata":{"id":"2JfnPnvMd8No"}},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"id":"rKgJ0IvLbtEL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1710218601987,"user_tz":420,"elapsed":1249,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"d78083e0-4315-4a75-d558-c998f1ca6a5b"},"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}]},{"cell_type":"code","source":["import os\n","import pandas as pd\n","from tqdm import tqdm\n","\n","parent_dir = '/content/drive/MyDrive/cs247project/data/train_data'\n","\n","# Initialize an empty DataFrame to store the combined data\n","df = pd.DataFrame()\n","\n","# Iterate through each folder (train1 through train7)\n","br = 0\n","br2 = 0\n","for folder_name in os.listdir(parent_dir):\n","    if br == 1:\n","      break\n","    folder_path = os.path.join(parent_dir, folder_name)\n","    br = 1\n","\n","    # Check if the path is a directory\n","    if os.path.isdir(folder_path):\n","        # Iterate through each CSV file in the folder\n","        for file_name in tqdm(os.listdir(folder_path), desc=f\"Processing {folder_name}\"):\n","            if file_name.endswith('.csv'):\n","\n","                file_path = os.path.join(folder_path, file_name)\n","\n","                # Read the CSV file into a DataFrame\n","                this_df = pd.read_csv(file_path)\n","\n","                # Concatenate the current DataFrame with the combined_data DataFrame\n","                df = pd.concat([df, this_df], ignore_index=True)\n","\n","# Print the combined DataFrame\n","print(df)\n"],"metadata":{"id":"zHZiYfQGcvYg","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1710219132792,"user_tz":420,"elapsed":107503,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"65f64665-d067-4e63-d1f0-7ba2daa2ca19"},"execution_count":11,"outputs":[{"output_type":"stream","name":"stderr","text":["Processing train1: 100%|██████████| 11/11 [01:44<00:00,  9.52s/it]\n"]},{"output_type":"stream","name":"stdout","text":["         page_view_start_time  \\\n","0               1586392478339   \n","1               1587264330576   \n","2               1586971453279   \n","3               1586446676383   \n","4               1587072791703   \n","...                       ...   \n","4867448         1586283443662   \n","4867449         1587236723732   \n","4867450         1586865909283   \n","4867451         1586273456619   \n","4867452         1587268134242   \n","\n","                                              user_id_hash  \\\n","0        939e2ccfb21e0ec9b0787c394dea1ae935369da0bcc597...   \n","1        decc9bcd3a7f2807721a35368b362782ec5bf4b4fb7ecf...   \n","2        30b6851a72026b837c7b33190ec7684763f36a8cd7722c...   \n","3        0b03f7d7bfc9d81e55ebeb994d572ca65cca353e7aaf85...   \n","4        b46cb1c9681e58c0a09b88b82c36913cbf3f710674eb39...   \n","...                                                    ...   \n","4867448  0d3c92f25f9375068a1b9215e86da67e0f09283e145af8...   \n","4867449  7ceacd0153187e3e4a0e398278c3b82e6c5d0eba4fdf1f...   \n","4867450  7a91b4737b70b51bc97da948df6c45db8f9f6d6c383cfb...   \n","4867451  241fa37669c8b94d574ee505062e2aa2101732aebd0e61...   \n","4867452  ba7ff429aa6cc3aca8359f431dc8eb954c801d2bacfe94...   \n","\n","                                            target_id_hash  \\\n","0        7349cb051df9c68532a2e4816aa1dc074141a7ce4a1236...   \n","1        415462704d29d1d343c78d2bb466402adc222f0969d5a9...   \n","2        1487501fb59aa7bfdc8744fc839e30cf115d9dbdb9fe5c...   \n","3        8058abb5518bb536d98c395013fb9081368ed89cca5980...   \n","4        d69f8bbef63a1c87fd253bfe1354894c06b194a2e6c30c...   \n","...                                                    ...   \n","4867448  8bb03e41f7a0d6d5bedfe5c287bd08680decac5726f8d7...   \n","4867449  6054d73fdc9a6de00cb5de7c2e6a5a2f7f88c355e9e013...   \n","4867450  44e949862d279458136ebf46b9e882fea3a8adaf93e4d4...   \n","4867451  f00d5db0ef399c8e6dfcc05987952c9ac9548aec26be4e...   \n","4867452  b535cf6896234d0ae93988b9f11b2be1a87777f4a8935f...   \n","\n","                                        syndicator_id_hash  \\\n","0        ebc206ae45aa38e5ffafafb3a91dc289d4a0838937c728...   \n","1        85d3c3f9981f84f4d10d01a9115962b99e00ef325ef20d...   \n","2        56657dd10b5a8391d470928f5f92192df442ec7b927bf3...   \n","3        daaee2d38f21a75a08d375e55802b3fdfc03cf21608ebd...   \n","4        2f216e3448659404205a319e9cada63f967871b2c3e0d5...   \n","...                                                    ...   \n","4867448  3275de7ceddde55973aa48e20befc04a40724ec730b70b...   \n","4867449  99c1c24deac297ef4989bc5a337066cf8dada55a093236...   \n","4867450  4428e3d0a6542eda13ebb862cb20c6317b4ab81509972c...   \n","4867451  76476668743f32c7e7d5b4e24647bbd845109057046a78...   \n","4867452  ec28de107e3ac04bc6c8a7aa122f57d4f7f45339c54d01...   \n","\n","                                          campaign_id_hash  \\\n","0        df4a08d5f84793cb72cb0bfdb50e82a7f6ce6cae5701d7...   \n","1        076a73951276d40a877a8fc065cfbdd1cc4af5685d63c7...   \n","2        e90bf004fa3fd9ac170c239e425dbfedbef0453b93a5dd...   \n","3        1633f7f86d0b74925000ec3ccb2662ae6e41a0de72669b...   \n","4        8171fbc15cbd58ba2a41db9445a84423ccff42cf91f035...   \n","...                                                    ...   \n","4867448  93056e3df041479734db7c02f7892f2168eaf4ee5fccab...   \n","4867449  694077ba582ace2afd7f9c09fc9f3b15baf1b79680fea2...   \n","4867450  b0bb57c38ec6f52f6464e2da3b96ec1152414badcdb2d3...   \n","4867451  9f96a8607cb7e8f151a579b7a9c8b78581d2fadf7c1abe...   \n","4867452  b935f6953252344787e8e9e3d5f3ab1899553844fd78dc...   \n","\n","         empiric_calibrated_recs  empiric_clicks target_item_taxonomy  \\\n","0                    13107.24400         24312.0              UNKNOWN   \n","1                   129166.30000        103160.0             BUSINESS   \n","2                    16507.39600         17779.0  ENTERTAINMENT~MUSIC   \n","3                      965.36127          5513.0          LIFE~HEALTH   \n","4                     4976.14800          1974.0             BUSINESS   \n","...                          ...             ...                  ...   \n","4867448                104.64570           583.0  ENTERTAINMENT~MUSIC   \n","4867449              11496.30200          8227.0             BUSINESS   \n","4867450                529.02026          9058.0               SPORTS   \n","4867451               3820.09900          2282.0                 TECH   \n","4867452               2866.78120         13827.0         LIFE~FASHION   \n","\n","                                         placement_id_hash  user_recs  ...  \\\n","0        2a1989e08355c7e9539e9b9a01c9a7f7869d75bf071b9b...       73.0  ...   \n","1        94a75567cd09d67c3f74d15ee8d82dea79030a5f30a0f4...       43.0  ...   \n","2        932fbd7955800fd5a36f071a7e74f18343c263cf4bd950...       61.0  ...   \n","3        3e169c2c7c217891ced33c35c10f01ba9b1a830e58ff24...      357.0  ...   \n","4        6207a672a60f19e0225b1ce438bfc5138af3f549787baa...      825.0  ...   \n","...                                                    ...        ...  ...   \n","4867448  970f1987570a4ece129c056ddf4de53b1f43b0882ea65c...      586.0  ...   \n","4867449  8f9cd7f2c10b4d944e448297755be91fde5a4e2f982e37...        0.0  ...   \n","4867450  3c1c8ea94e66caefc804eba418aa893f0d8fc361ad86e5...      602.0  ...   \n","4867451  b9c492d9786b9bd1958f356f0828fe08812a0ee00c6660...      168.0  ...   \n","4867452  3946bcf55580d76c92c83f42a44bdf62e1e54740b48646...     1582.0  ...   \n","\n","                                            source_id_hash  source_item_type  \\\n","0        ec15d7e4d8e2717a592d339ce79beef7249bfc598278b8...              text   \n","1        389b69708cf58731e78e2e023a8c608d886fe5aebc0e23...          category   \n","2        afd2bead96464ae13fb61ba573ea503355865336dac9bf...             photo   \n","3        12811e3e09990653423da6481b0a67215096ff5bbd2fba...              text   \n","4        5e7a6ed11cdc1472ea91031bc66078c8ecd5283e2863d2...              text   \n","...                                                    ...               ...   \n","4867448  4c7987d659b97d2007c45bb3c9fd903b135a2ef18130f7...          homepage   \n","4867449  389b69708cf58731e78e2e023a8c608d886fe5aebc0e23...          category   \n","4867450  5e5db386dc383e82c522aac0e4de9bb13a5ee720afd5c0...              text   \n","4867451  3db83af1dc21760b730e737568bb46294cff9985d9e6fd...              text   \n","4867452  8b93a436499a9e8fc75483756c973323b73a7c3b281cba...             photo   \n","\n","        browser_platform os_family country_code region  day_of_week  \\\n","0                   PHON         2           US     FL            3   \n","1                   DESK         6           US     TX            6   \n","2                   PHON         2           US     TX            3   \n","3                   DESK         6           US     FL            4   \n","4                   DESK         6           US     CO            4   \n","...                  ...       ...          ...    ...          ...   \n","4867448             DESK         6           US     MI            2   \n","4867449             DESK         6           US     MO            6   \n","4867450             DESK         6           US     OH            2   \n","4867451             PHON         3           US     WI            2   \n","4867452             DESK         6           US     WA            6   \n","\n","        time_of_day gmt_offset  is_click  \n","0                20       -400       1.0  \n","1                21       -500       0.0  \n","2                12       -500       1.0  \n","3                11       -400       0.0  \n","4                15       -600       0.0  \n","...             ...        ...       ...  \n","4867448          14       -400       1.0  \n","4867449          14       -500       0.0  \n","4867450           8       -400       1.0  \n","4867451          10       -500       0.0  \n","4867452          20       -700       1.0  \n","\n","[4867453 rows x 23 columns]\n"]}]},{"cell_type":"code","source":["import pandas as pd\n","import numpy as np\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","%matplotlib inline\n","sns.set(color_codes=True)"],"metadata":{"id":"ukPb8PqsNyhy","executionInfo":{"status":"ok","timestamp":1710218604759,"user_tz":420,"elapsed":1263,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":["## Get basic info about features"],"metadata":{"id":"E386-ndKqdCr"}},{"cell_type":"code","source":["df.head(5)"],"metadata":{"id":"JxGJA1qsQCLr","colab":{"base_uri":"https://localhost:8080/","height":53},"executionInfo":{"status":"ok","timestamp":1710218604759,"user_tz":420,"elapsed":13,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"4ce7fb4e-0ec8-4219-ff6c-ca1c51fcec4e"},"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Empty DataFrame\n","Columns: []\n","Index: []"],"text/html":["\n","  <div id=\"df-fd4146a8-1238-4d83-8883-0d7bcea67436\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fd4146a8-1238-4d83-8883-0d7bcea67436')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-fd4146a8-1238-4d83-8883-0d7bcea67436 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-fd4146a8-1238-4d83-8883-0d7bcea67436');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df","summary":"{\n  \"name\": \"df\",\n  \"rows\": 0,\n  \"fields\": []\n}"}},"metadata":{},"execution_count":4}]},{"cell_type":"code","source":["df.dtypes"],"metadata":{"id":"gDUBEg1DQJIs","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1710218604759,"user_tz":420,"elapsed":10,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"51b7835d-13d4-401b-fd29-8968a1097fce"},"execution_count":5,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Series([], dtype: object)"]},"metadata":{},"execution_count":5}]},{"cell_type":"code","source":["df.shape"],"metadata":{"id":"DFGdLcVSQogy","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1710218604759,"user_tz":420,"elapsed":8,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"5a06bc36-e356-42ca-9218-79e18239bdf8"},"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(0, 0)"]},"metadata":{},"execution_count":6}]},{"cell_type":"code","source":["df.count()"],"metadata":{"id":"lfjfmrCaQrN5","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1710218604759,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"153ebce4-49f0-4bc9-c160-23d43645f59e"},"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Series([], dtype: int64)"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["# Visualize is_click, make sure the classes are balanced\n","plt.figure(figsize=(8, 5))\n","sns.countplot(x='is_click', data=df)\n","plt.title('Distribution of Clicks')\n","plt.xlabel('is_click')\n","plt.ylabel('Count')\n","plt.show()"],"metadata":{"id":"Ey4cb8k0qiON","colab":{"base_uri":"https://localhost:8080/","height":436},"executionInfo":{"status":"error","timestamp":1710218604984,"user_tz":420,"elapsed":232,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"c94fb979-d1fa-4b79-83a8-a52d4b474fbe"},"execution_count":8,"outputs":[{"output_type":"error","ename":"ValueError","evalue":"Could not interpret value `is_click` for `x`. An entry with this name does not appear in `data`.","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)","\u001b[0;32m<ipython-input-8-8cb450e45995>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Visualize is_click, make sure the classes are balanced\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0msns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcountplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'is_click'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Distribution of Clicks'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxlabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'is_click'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/seaborn/categorical.py\u001b[0m in \u001b[0;36mcountplot\u001b[0;34m(data, x, y, hue, order, hue_order, orient, color, palette, saturation, fill, hue_norm, stat, width, dodge, gap, log_scale, native_scale, formatter, legend, ax, **kwargs)\u001b[0m\n\u001b[1;32m   2628\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Cannot pass values for both `x` and `y`.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2629\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2630\u001b[0;31m     p = _CategoricalAggPlotter(\n\u001b[0m\u001b[1;32m   2631\u001b[0m         \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2632\u001b[0m         \u001b[0mvariables\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/seaborn/categorical.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, variables, order, orient, require_numeric, color, legend)\u001b[0m\n\u001b[1;32m     64\u001b[0m     ):\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0;31m# This method takes care of some bookkeeping that is necessary because the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/seaborn/_base.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, variables)\u001b[0m\n\u001b[1;32m    632\u001b[0m         \u001b[0;31m# information for numeric axes would be information about log scales.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_var_ordered\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m\"x\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"y\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m}\u001b[0m  \u001b[0;31m# alt., used DefaultDict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 634\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massign_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    635\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    636\u001b[0m         \u001b[0;31m# TODO Lots of tests assume that these are called to initialize the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/seaborn/_base.py\u001b[0m in \u001b[0;36massign_variables\u001b[0;34m(self, data, variables)\u001b[0m\n\u001b[1;32m    677\u001b[0m             \u001b[0;31m# to centralize / standardize data consumption logic.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    678\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_format\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"long\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 679\u001b[0;31m             \u001b[0mplot_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPlotData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    680\u001b[0m             \u001b[0mframe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplot_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    681\u001b[0m             \u001b[0mnames\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplot_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/seaborn/_core/data.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, variables)\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhandle_data_source\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0mframe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_assign_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mframe\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/seaborn/_core/data.py\u001b[0m in \u001b[0;36m_assign_variables\u001b[0;34m(self, data, variables)\u001b[0m\n\u001b[1;32m    230\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m                     \u001b[0merr\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m\"An entry with this name does not appear in `data`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mValueError\u001b[0m: Could not interpret value `is_click` for `x`. An entry with this name does not appear in `data`."]},{"output_type":"display_data","data":{"text/plain":["<Figure size 800x500 with 0 Axes>"]},"metadata":{}}]},{"cell_type":"markdown","source":["## Remove null values"],"metadata":{"id":"pPQDqkNPqpzx"}},{"cell_type":"code","source":["print(df.isnull().sum())"],"metadata":{"id":"hvgyMciQQyut","executionInfo":{"status":"aborted","timestamp":1710218604984,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df = df.dropna()    # Dropping the missing values.\n","df.count()"],"metadata":{"id":"arRjblOoQ6zq","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(df.isnull().sum())"],"metadata":{"id":"fQsRgPBmQ_vC","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Check for outliers"],"metadata":{"id":"oaDVFa0Rqsto"}},{"cell_type":"code","source":["cols = ['page_view_start_time', 'empiric_clicks', 'user_recs', 'time_of_day'] # can add/remove from this as needed\n","\n","for col in cols:\n","      # Create a boxplot for the current column\n","      sns.boxplot(x=df[col])\n","\n","      # Set plot title with the column name\n","      plt.title(f'Boxplot for {col}')\n","\n","      # Show the plot\n","      plt.show()"],"metadata":{"id":"0EZjiOWqRDr2","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["unique_user_ids = df['user_id_hash'].nunique()\n","print(unique_user_ids)"],"metadata":{"id":"D8LaOWGakEVJ","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["target_id_hash = df['target_id_hash'].nunique()\n","print(target_id_hash)"],"metadata":{"id":"u8RnKBqck7Ow","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Identify most important features:"],"metadata":{"id":"jLHR3pACpR0g"}},{"cell_type":"code","source":["import torch\n","\n","from spotlight.factorization.explicit import ExplicitFactorizationModel\n","from spotlight.datasets.movielens import get_movielens_dataset\n","\n","\n","model = ExplicitFactorizationModel(loss='regression',\n","                                   embedding_dim=128,  # latent dimensionality\n","                                   n_iter=10,  # number of epochs of training\n","                                   batch_size=1024,  # minibatch size\n","                                   l2=1e-9,  # strength of L2 regularization\n","                                   learning_rate=1e-3,\n","                                   use_cuda=torch.cuda.is_available())\n"],"metadata":{"id":"UuIa_8JmzE1u","executionInfo":{"status":"ok","timestamp":1710219334324,"user_tz":420,"elapsed":5506,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":12,"outputs":[]},{"cell_type":"code","source":["from spotlight.cross_validation import random_train_test_split\n","from spotlight.interactions import Interactions\n","import numpy as np\n","\n","# Assuming 'user_id_hash' is your user ID column and 'target_id_hash' is your item ID column\n","# and 'is_click' is your interaction/rating column (you may need to convert these to integer indices)\n","user_ids = df['user_id_hash'].astype('category').cat.codes.values\n","item_ids = df['target_id_hash'].astype('category').cat.codes.values\n","ratings = df['is_click'].values\n","\n","# Create an Interactions object\n","interactions = Interactions(user_ids, item_ids, ratings)\n","\n","# Now you can split it using spotlight's function\n","\n","train, test = random_train_test_split(interactions, random_state=np.random.RandomState(42))\n","\n","print('Split into \\n {} and \\n {}.'.format(train, test))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zPA7YcrRzlXO","executionInfo":{"status":"ok","timestamp":1710219370775,"user_tz":420,"elapsed":34197,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"95e5c049-b0d0-4ec8-dde6-88fed891cd4b"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["Split into \n"," <Interactions dataset (3950069 users x 61763 items x 3893962 interactions)> and \n"," <Interactions dataset (3950069 users x 61763 items x 973491 interactions)>.\n"]}]},{"cell_type":"code","source":["model.fit(train, verbose=True)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":144},"id":"fHxmQgOpzmJa","executionInfo":{"status":"error","timestamp":1710219539785,"user_tz":420,"elapsed":173,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}},"outputId":"ec9da96b-8f7e-4b32-83b9-a582593ff77a"},"execution_count":1,"outputs":[{"output_type":"error","ename":"NameError","evalue":"name 'model' is not defined","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-1-86633c1a9f58>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"]}]},{"cell_type":"code","source":["from spotlight.evaluation import rmse_score\n","\n","train_rmse = rmse_score(model, train)\n","test_rmse = rmse_score(model, test)\n","\n","print('Train RMSE {:.3f}, test RMSE {:.3f}'.format(train_rmse, test_rmse))"],"metadata":{"id":"kvD01M8pzoE4","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from sklearn.ensemble import RandomForestClassifier\n","\n","numeric_features = df.select_dtypes(include=['int64', 'float64']).columns.tolist()\n","df_numeric_features_only = df.loc[:, numeric_features]\n","\n","X = df_numeric_features_only.drop('is_click', axis=1)\n","y = df_numeric_features_only['is_click']\n","\n","# Create and train the RandomForestClassifier\n","model = RandomForestClassifier(random_state=42)\n","model.fit(X, y)\n","\n","# Get feature importances\n","feature_importances = model.feature_importances_\n","\n","# Combine with feature names\n","feature_importance_dict = dict(zip(X.columns, feature_importances))\n","\n","# Sort and display\n","sorted_features = sorted(feature_importance_dict.items(), key=lambda x: x[1], reverse=True)\n","print(sorted_features)\n","\n"],"metadata":{"id":"oshetcztmbwr","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Plot the two most important features\n","plt.figure(figsize=(10, 6))\n","\n","# Scatter plot with custom colors based on is_click\n","colors = df['is_click'].map({0: 'red', 1: 'green'})\n","plt.scatter(df['empiric_calibrated_recs'], df['empiric_clicks'], c=colors)\n","\n","# Adding labels and title\n","plt.title('Scatter Plot: empiric_calibrated_recs vs empiric_clicks')\n","plt.xlabel('empiric_calibrated_recs')\n","plt.ylabel('empiric_clicks')\n","\n","plt.show()"],"metadata":{"id":"7wsZ5Bw9p9_7","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Plot the some of the less important features\n","plt.figure(figsize=(10, 6))\n","\n","# Scatter plot with custom colors based on is_click\n","colors = df['is_click'].map({0: 'red', 1: 'green'})\n","plt.scatter(df['user_target_recs'], df['user_recs'], c=colors)\n","\n","# Adding labels and title\n","plt.title('Scatter Plot: user_target_recs vs user_recs')\n","plt.xlabel('user_target_recs')\n","plt.ylabel('user_recs')\n","\n","plt.show()\n"],"metadata":{"id":"_OddFMVgr12-","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# https://www.kaggle.com/code/imoore/intro-to-exploratory-data-analysis-eda-in-python"],"metadata":{"id":"p_uXNzvRRVGw","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Baseline Classifiers"],"metadata":{"id":"AL5HmaGVw7lh"}},{"cell_type":"code","source":["from sklearn.model_selection import train_test_split\n","from sklearn.metrics import accuracy_score\n","\n","# All our baselines only use the numerical features in the dataset\n","X = df_numeric_features_only.drop('is_click', axis=1)  # Features\n","y = df_numeric_features_only['is_click']  # Target variable\n","\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"],"metadata":{"id":"JMXKKpk9yUcW","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Logistic Regression"],"metadata":{"id":"fECDsNFNx2fG"}},{"cell_type":"code","source":["from sklearn.linear_model import LogisticRegression\n","\n","# Create a Logistic Regression model\n","model = LogisticRegression(random_state=42)\n","\n","# Train the model on the training set\n","model.fit(X_train, y_train)\n","\n","# Make predictions on the test set\n","y_pred = model.predict(X_test)\n","\n","# Evaluate the model\n","accuracy = accuracy_score(y_test, y_pred)\n","\n","print(f\"Accuracy: {accuracy:.2f}\")"],"metadata":{"id":"AYbO8trPxnya","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Decision Tree"],"metadata":{"id":"y0KxBUUmtZqS"}},{"cell_type":"code","source":["from sklearn.tree import DecisionTreeClassifier\n","from sklearn.metrics import accuracy_score\n","from sklearn import tree\n","\n","# Create a decision tree classifier\n","clf = DecisionTreeClassifier(random_state=42)\n","\n","# Train the classifier on the training set\n","clf.fit(X_train, y_train)\n","\n","# Make predictions on the test set\n","y_pred = clf.predict(X_test)\n","\n","# Evaluate the model\n","accuracy = accuracy_score(y_test, y_pred)\n","\n","print(f\"Accuracy: {accuracy:.2f}\")"],"metadata":{"id":"WH3ewgeWtdDG","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Random Forest"],"metadata":{"id":"iC6FrHGUw4fF"}},{"cell_type":"code","source":["from sklearn.model_selection import train_test_split\n","from sklearn.metrics import accuracy_score\n","import matplotlib.pyplot as plt\n","\n","# Create a Random Forest classifier\n","clf = RandomForestClassifier(random_state=42)\n","\n","# Train the classifier on the training set\n","clf.fit(X_train, y_train)\n","\n","# Make predictions on the test set\n","y_pred = clf.predict(X_test)\n","\n","# Evaluate the model\n","accuracy = accuracy_score(y_test, y_pred)\n","\n","print(f\"Accuracy: {accuracy:.2f}\")\n"],"metadata":{"id":"Uc4EdUwxwhzc","executionInfo":{"status":"aborted","timestamp":1710218604985,"user_tz":420,"elapsed":7,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Perceptron"],"metadata":{"id":"YvE1HRINzL6d"}},{"cell_type":"code","source":["import tensorflow as tf\n","from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n","from sklearn.preprocessing import StandardScaler\n","\n","# Standardize the features\n","scaler = StandardScaler()\n","X_train_scaled = scaler.fit_transform(X_train)\n","X_test_scaled = scaler.transform(X_test)\n","\n","# Define one-layer neural network\n","model = tf.keras.Sequential([\n","    tf.keras.layers.Dense(1, activation='sigmoid', input_shape=(X_train_scaled.shape[1],))\n","])\n","\n","# Compile the model\n","model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n","\n","# Train the model\n","model.fit(X_train_scaled, y_train, epochs=10, batch_size=32, validation_data=(X_test_scaled, y_test))\n","\n","# Evaluate the model\n","y_pred_probs = model.predict(X_test_scaled)\n","y_pred = (y_pred_probs > 0.5).astype(int)\n","\n","accuracy = accuracy_score(y_test, y_pred)\n","conf_matrix = confusion_matrix(y_test, y_pred)\n","classification_rep = classification_report(y_test, y_pred)\n","\n","print(f\"Accuracy: {accuracy:.2f}\")\n","print(\"Confusion Matrix:\\n\", conf_matrix)\n","print(\"Classification Report:\\n\", classification_rep)\n"],"metadata":{"id":"-AWzrqIAyzI3","executionInfo":{"status":"aborted","timestamp":1710218604986,"user_tz":420,"elapsed":8,"user":{"displayName":"UTKARSH LAL","userId":"01578989176710731084"}}},"execution_count":null,"outputs":[]}]}