From 35cf2f4fba2a0fa30c35eaead2f1fd897337c97f Mon Sep 17 00:00:00 2001 From: Inga Date: Tue, 24 Oct 2023 00:24:35 +0000 Subject: [PATCH] implemented osm client --- package-lock.json | 91 ++++++++++++++++++++++++++++++- package.json | 6 ++ src/clients/geocoding/osm.spec.ts | 30 ++++++++++ src/clients/geocoding/osm.ts | 62 +++++++++++++++++++++ src/utils/math.spec.ts | 13 +++++ src/utils/math.ts | 10 ++++ 6 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 src/clients/geocoding/osm.spec.ts create mode 100644 src/clients/geocoding/osm.ts create mode 100644 src/utils/math.spec.ts create mode 100644 src/utils/math.ts diff --git a/package-lock.json b/package-lock.json index 40e87cd..b4ef342 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,10 @@ "@nestjs/common": "^10.0.0", "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", + "lodash": "^4.17.21", + "node-fetch": "^2.7.0", + "node-geocoder": "^4.2.0", + "p-queue": "^6.6.2", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1" }, @@ -22,7 +26,9 @@ "@tsconfig/strictest": "^2.0.2", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", + "@types/lodash": "^4.14.200", "@types/node": "^20.3.1", + "@types/node-geocoder": "^4.2.5", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", @@ -2014,6 +2020,12 @@ "integrity": "sha512-U3PUjAudAdJBeC2pgN8uTIKgxrb4nlDF3SF0++EldXQvQBGkpFZMSnwQiIoDU77tv45VgNkl/L4ouD+rEomujw==", "dev": true }, + "node_modules/@types/lodash": { + "version": "4.14.200", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.200.tgz", + "integrity": "sha512-YI/M/4HRImtNf3pJgbF+W6FrXovqj+T+/HpENLTooK9PnkacBsDpeP3IpHab40CClUfhNmdM2WTNP2sa2dni5Q==", + "dev": true + }, "node_modules/@types/mime": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.4.tgz", @@ -2029,6 +2041,26 @@ "undici-types": "~5.25.1" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-lX17GZVpJ/fuCjguZ5b3TjEbSENxmEk1B2z02yoXSK9WMEWRivhdSY73wWMn6bpcCDAOh6qAdktpKHIlkDk2lg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/node-geocoder": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/@types/node-geocoder/-/node-geocoder-4.2.5.tgz", + "integrity": "sha512-dyKHe2mx695bgkEuWiSMUdVHll+Y626W1fXYAU3qE9Gwgrl4A5J8pIUUyC3mkjpA73C+QX6lOqxb2zfAOliW9g==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/node-fetch": "^2" + } + }, "node_modules/@types/parse-json": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.1.tgz", @@ -2857,6 +2889,11 @@ "node": ">= 6" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "node_modules/body-parser": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", @@ -4139,6 +4176,11 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -6015,8 +6057,7 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash.memoize": { "version": "4.1.2", @@ -6332,6 +6373,18 @@ } } }, + "node_modules/node-geocoder": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/node-geocoder/-/node-geocoder-4.2.0.tgz", + "integrity": "sha512-ZvUiOHWHLVGlrYPvazXj+VQ9oK+EOMinh/5vWoBwOQiV0eCU7GPtEWrMRNnKvmBVOzZHb1eFe9uoKMFeyygMVA==", + "dependencies": { + "bluebird": "^3.5.2", + "node-fetch": "^2.6.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -6499,6 +6552,14 @@ "node": ">=0.10.0" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -6529,6 +6590,32 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", diff --git a/package.json b/package.json index 6fcf0a0..f3037bc 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,10 @@ "@nestjs/common": "^10.0.0", "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", + "lodash": "^4.17.21", + "node-fetch": "^2.7.0", + "node-geocoder": "^4.2.0", + "p-queue": "^6.6.2", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1" }, @@ -33,7 +37,9 @@ "@tsconfig/strictest": "^2.0.2", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", + "@types/lodash": "^4.14.200", "@types/node": "^20.3.1", + "@types/node-geocoder": "^4.2.5", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", diff --git a/src/clients/geocoding/osm.spec.ts b/src/clients/geocoding/osm.spec.ts new file mode 100644 index 0000000..d002151 --- /dev/null +++ b/src/clients/geocoding/osm.spec.ts @@ -0,0 +1,30 @@ +import { createOsmClient } from './osm'; + +const sampleAddresses = [ + { + address: 'Hamburger Str. 273A, 38114 Braunschweig, Germany', + latitude: 52.28, + longitude: 10.52, + }, + { + address: 'Carrer de Tapioles, 47, 08004 Barcelona, Spain', + latitude: 41.37, + longitude: 2.16, + }, + { + address: 'Dumlupınar Sk. No:5, 34710 Kadıköy/İstanbul, Türkiye', + latitude: 40.99, + longitude: 29.02, + }, +]; + +describe('createOsmClient', () => { + const client = createOsmClient(); + for (const sampleAddress of sampleAddresses) { + it(`resolves ${sampleAddress.address}`, async () => { + const result = await client.geocode(sampleAddress.address); + expect(result.latitude).toBeCloseTo(sampleAddress.latitude, 2); + expect(result.longitude).toBeCloseTo(sampleAddress.longitude, 2); + }); + } +}); diff --git a/src/clients/geocoding/osm.ts b/src/clients/geocoding/osm.ts new file mode 100644 index 0000000..12e28cd --- /dev/null +++ b/src/clients/geocoding/osm.ts @@ -0,0 +1,62 @@ +import { compact } from 'lodash'; +import NodeGeocoder from 'node-geocoder'; +import fetch from 'node-fetch'; +import PQueue from 'p-queue'; +import { mean } from '../../utils/math'; + +export const createOsmClient = () => { + const geocoder = NodeGeocoder({ + provider: 'openstreetmap', + fetch: (url, options) => { + return fetch(url, { + ...options, + headers: { + ...options?.headers, + 'user-agent': + 'test-assignment-parcellab by Inga. This agent is only supposed to send a few requests in two weeks starting on October 23th 2023, and none after that', + }, + }); + }, + }); + + // Rate limit according to https://operations.osmfoundation.org/policies/nominatim/ + const queue = new PQueue({ + interval: 1000, + intervalCap: 1, + timeout: 2000, + throwOnTimeout: true, + }); + + return { + geocode: async (query: string) => { + const result = await queue.add(() => geocoder.geocode(query)); + + if (!result.length) { + throw new Error('No results found'); + } + + const meanLatitude = mean( + compact(result.map(({ latitude }) => latitude)), + ); + const meanLongitude = mean( + compact(result.map(({ longitude }) => longitude)), + ); + if ( + !result.every( + ({ latitude, longitude }) => + latitude && + longitude && + Math.abs(latitude - meanLatitude) < 0.01 && + Math.abs(longitude - meanLongitude) < 0.01, + ) + ) { + throw new Error('Ambiguous address'); + } + + return { + latitude: meanLatitude, + longitude: meanLongitude, + }; + }, + }; +}; diff --git a/src/utils/math.spec.ts b/src/utils/math.spec.ts new file mode 100644 index 0000000..66e072a --- /dev/null +++ b/src/utils/math.spec.ts @@ -0,0 +1,13 @@ +import { mean } from './math'; + +describe('mean', () => { + it('computes mean for one value correctly', () => { + expect(mean([10])).toBe(10); + }); + it('computes mean for two value correctly', () => { + expect(mean([10, 11])).toBe(10.5); + }); + it('computes mean for tem values correctly', () => { + expect(mean([10, 11, 12, 13, 14, 15, 16, 17, 18, 19])).toBe(14.5); + }); +}); diff --git a/src/utils/math.ts b/src/utils/math.ts new file mode 100644 index 0000000..23026c7 --- /dev/null +++ b/src/utils/math.ts @@ -0,0 +1,10 @@ +export const mean = (values: number[]) => { + let currentMean = 0; + let count = 0; + for (const value of values) { + count++; + currentMean += (value - currentMean) / count; + } + + return currentMean; +};