-
Notifications
You must be signed in to change notification settings - Fork 463
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Programatically add dots to map
Instead of having a hardcoded list of dots in our map, we're now collecting the latitude/longitude information from the information we're already collecting from everyone working at PostHog. We're doing some geocoding to transform that information into lat/long pairs. There are some problems with the fact that there isn't much uniformity in the way everyone stores their information, so some dots are extremely innacurate. I've attempted the following pairs and they don't work that well - Location - Location, Country - Location | Country I ultimately settled with the last one because it looks the most accurate - but there are errors still. My suggestion: we should suggest everyone to fill their "Location" field with either a "city, country" tuple or simply the long form of their country name, which **should** make the map more accurate.
- Loading branch information
1 parent
e63a256
commit ef6f749
Showing
6 changed files
with
342 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| const MapboxClient = require('@mapbox/mapbox-sdk') | ||
| const { chunk } = require('lodash') | ||
|
|
||
| const locationForProfile = (profile) => { | ||
| return profile.location ? | ||
| { q: profile.location, types: ['place', 'region', 'country'] } : | ||
| { q: profile.country, types: ['country'] } | ||
| } | ||
|
|
||
| const sourceNodes = async (options, pluginOptions) => { | ||
| const { actions: { createNode }, createNodeId, createContentDigest, getNodes, reporter } = options | ||
| const { mapboxToken } = pluginOptions | ||
|
|
||
| if (!mapboxToken) { | ||
| reporter.panic('You must provide a Mapbox access token') | ||
| return | ||
| } | ||
|
|
||
| // Initialize Mapbox client | ||
| const mapboxClient = new MapboxClient({ accessToken: mapboxToken }) | ||
|
|
||
| // Get all Squeak profiles directly from nodes | ||
| // Implement the filter below to guarantee we're not processing profiles that don't have a team | ||
|
|
||
| const profiles = getNodes() | ||
| .filter(node => | ||
| node.internal.type === 'SqueakProfile' && // For all Squeak profiles | ||
| node.teams?.data?.length > 0 && // Implement the following to avoid old profiles: filter: { teams: { data: { elemMatch: { id: { ne: null } } } } } | ||
| (node.location || node.country) // Only process profiles with a location or country | ||
| ) | ||
| .map(node => ({ | ||
| id: node.id, | ||
| location: node.location, | ||
| country: node.country, | ||
| })) | ||
|
|
||
| const BATCH_SIZE = 50 | ||
| const batches = chunk(profiles, BATCH_SIZE) | ||
|
|
||
| reporter.info(`Processing ${profiles.length} locations in ${batches.length} batches`) | ||
|
|
||
| const locations = [] | ||
|
|
||
| for (const [index, batch] of batches.entries()) { | ||
| reporter.info(`Processing batch ${index + 1}/${batches.length}`) | ||
|
|
||
| try { | ||
| const response = await mapboxClient.createRequest({ | ||
| method: 'POST', | ||
| path: '/search/geocode/v6/batch', | ||
| body: batch.map(locationForProfile) | ||
| }).send() | ||
|
|
||
| console.log(JSON.stringify(response.body.batch, null, 2)) | ||
|
|
||
| // Match results with original profiles | ||
| response.body.batch.forEach((entry, i) => { | ||
| if (entry.features.length > 0) { | ||
| const [longitude, latitude] = entry.features[0]?.geometry?.coordinates | ||
| if (longitude && latitude) { | ||
| locations.push({ | ||
| profileId: batch[i].id, | ||
| location: locationForProfile(batch[i]).q, | ||
| coordinates: { | ||
| latitude, | ||
| longitude | ||
| } | ||
| }) | ||
| } | ||
| } | ||
| }) | ||
| } catch (error) { | ||
| reporter.warn(`Failed to process batch ${index + 1}: ${error.message}`) | ||
| } | ||
| } | ||
|
|
||
| reporter.info(`Successfully processed ${locations.length} locations`) | ||
|
|
||
| // Create nodes directly here, not in a separate function | ||
| locations.forEach(location => { | ||
| const nodeContent = { | ||
| ...location, | ||
| } | ||
|
|
||
| const nodeMeta = { | ||
| id: createNodeId(`mapbox-location-${location.profileId}`), | ||
| parent: null, | ||
| children: [], | ||
| internal: { | ||
| type: `MapboxLocation`, | ||
| content: JSON.stringify(nodeContent), | ||
| contentDigest: createContentDigest(nodeContent), | ||
| }, | ||
| } | ||
|
|
||
| createNode({ ...nodeContent, ...nodeMeta }) | ||
| }) | ||
| } | ||
|
|
||
| module.exports = { sourceNodes } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| { | ||
| "name": "gatsby-mapbox-locations", | ||
| "version": "1.0.0", | ||
| "description": "Gatsby plugin to fetch location coordinates from Mapbox according to a Squeak source", | ||
| "main": "gatsby-node.js", | ||
| "dependencies": { | ||
| "@mapbox/mapbox-sdk": "^0.15.3" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.