chore(amsterdump): Remove project
This has looong outlived its purpose. Change-Id: I66cff438dd05f0113b4a9a9bf9023d0f72238c9f Reviewed-on: https://cl.tvl.fyi/c/depot/+/2651 Reviewed-by: tazjin <mail@tazj.in> Tested-by: BuildkiteCI
This commit is contained in:
Vincent Ambo
tazjin
parent
aaaadbbcdc
commit
63e1706096
4 changed files with 0 additions and 2472 deletions
|
|
@ -1,13 +0,0 @@
|
||||||
{ depot, ... }:
|
|
||||||
|
|
||||||
depot.nix.buildGo.program {
|
|
||||||
name = "amsterdump";
|
|
||||||
srcs = [
|
|
||||||
./main.go
|
|
||||||
];
|
|
||||||
|
|
||||||
deps = with depot.third_party; [
|
|
||||||
# gopkgs."golang.org".x.oauth2.google
|
|
||||||
gopkgs."googlemaps.github.io".maps
|
|
||||||
];
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,108 +0,0 @@
|
||||||
// Amsterdump is a small program that populates a BigQuery table with
|
|
||||||
// a matrix of origin points scattered around Amsterdam and each
|
|
||||||
// respective points travel time to a given destination.
|
|
||||||
//
|
|
||||||
// The two destinations used here are the Schiphol Airport and
|
|
||||||
// Amsterdam Central station.
|
|
||||||
//
|
|
||||||
// To accomplish this the Google Maps Distance Matrix API [1] is
|
|
||||||
// queried with the points. A visualisation is later done using
|
|
||||||
// BigQuery GeoViz[2].
|
|
||||||
//
|
|
||||||
// [1]: https://developers.google.com/maps/documentation/distance-matrix/start#quotas
|
|
||||||
// [2]: https://bigquerygeoviz.appspot.com/
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"googlemaps.github.io/maps"
|
|
||||||
)
|
|
||||||
|
|
||||||
func failOn(err error, msg string) {
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalln(msg, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type LocationResult struct {
|
|
||||||
Address string `json:"address"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
Schiphol *maps.DistanceMatrixElement `json:"schiphol"`
|
|
||||||
Centraal *maps.DistanceMatrixElement `json:"centraal"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Listing struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
Address string `json:"address"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func requestMatrix(ctx context.Context, client *maps.Client, listings []Listing) {
|
|
||||||
origins := make([]string, len(listings))
|
|
||||||
for i, l := range listings {
|
|
||||||
origins[i] = l.Address
|
|
||||||
}
|
|
||||||
|
|
||||||
request := &maps.DistanceMatrixRequest{
|
|
||||||
Mode: maps.TravelModeTransit,
|
|
||||||
Units: maps.UnitsMetric,
|
|
||||||
Origins: origins,
|
|
||||||
|
|
||||||
Destinations: []string{
|
|
||||||
"Schiphol Airport",
|
|
||||||
"Amsterdam Centraal",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.DistanceMatrix(ctx, request)
|
|
||||||
failOn(err, "could not retrieve distance matrix:")
|
|
||||||
|
|
||||||
for idx, addr := range response.OriginAddresses {
|
|
||||||
result := LocationResult{
|
|
||||||
Address: addr,
|
|
||||||
URL: listings[idx].URL,
|
|
||||||
Schiphol: response.Rows[idx].Elements[0],
|
|
||||||
Centraal: response.Rows[idx].Elements[1],
|
|
||||||
}
|
|
||||||
|
|
||||||
j, _ := json.Marshal(result)
|
|
||||||
fmt.Println(string(j))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
var listings []Listing
|
|
||||||
input, err := ioutil.ReadFile("fun/amsterdump/input.json")
|
|
||||||
failOn(err, "could not read input file:")
|
|
||||||
|
|
||||||
err = json.Unmarshal(input, &listings)
|
|
||||||
failOn(err, "could not deserialise listings:")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
apiKey := os.Getenv("MAPS_API_KEY")
|
|
||||||
if apiKey == "" {
|
|
||||||
log.Fatalln("API key must be supplied via MAPS_API_KEY")
|
|
||||||
}
|
|
||||||
|
|
||||||
client, err := maps.NewClient(maps.WithAPIKey(apiKey))
|
|
||||||
failOn(err, "could not create Google Maps API client:")
|
|
||||||
|
|
||||||
var chunk []Listing
|
|
||||||
for _, l := range listings {
|
|
||||||
if len(chunk) == 25 {
|
|
||||||
requestMatrix(ctx, client, chunk)
|
|
||||||
chunk = []Listing{}
|
|
||||||
} else {
|
|
||||||
chunk = append(chunk, l)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(chunk) > 1 {
|
|
||||||
requestMatrix(ctx, client, chunk)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
;; Scraping funda.nl (this file is just notes and snippets, not full code)
|
|
||||||
;;
|
|
||||||
;; Begin by copying whole page into buffer (out of inspect element
|
|
||||||
;; because encoding is difficult)
|
|
||||||
|
|
||||||
(beginning-of-buffer)
|
|
||||||
|
|
||||||
;; zap everything that isn't a relevant result
|
|
||||||
(keep-lines "data-object-url-tracking\\|img alt")
|
|
||||||
|
|
||||||
;; mark all spans, move them to the end of the buffer
|
|
||||||
(cl-letf (((symbol-function 'read-regexp)
|
|
||||||
(lambda (&rest _) "</span>")))
|
|
||||||
(mc/mark-all-in-region-regexp (point-min) (point-max)))
|
|
||||||
|
|
||||||
;; mark all images lines (these contain street addresses for things
|
|
||||||
;; with images), clear up and join with previous
|
|
||||||
;;
|
|
||||||
;; mark all: data-image-error-fallback
|
|
||||||
|
|
||||||
;; delete all lines that don't either contain a span or an img tag
|
|
||||||
;; (there are duplicates)
|
|
||||||
(keep-lines "span class\\|img alt")
|
|
||||||
|
|
||||||
;; do some manual cleanup from the hrefs and done
|
|
||||||
Loading…
Reference in a new issue