chore(amsterdump): Remove project
This has looong outlived its purpose. Change-Id: I66cff438dd05f0113b4a9a9bf9023d0f72238c9f Reviewed-on: https://cl.tvl.fyi/c/depot/+/2651 Reviewed-by: tazjin <mail@tazj.in> Tested-by: BuildkiteCI
This commit is contained in:
Vincent Ambo
tazjin
parent
aaaadbbcdc
commit
63e1706096
4 changed files with 0 additions and 2472 deletions
|
|
@ -1,13 +0,0 @@
|
|||
{ depot, ... }:
|
||||
|
||||
depot.nix.buildGo.program {
|
||||
name = "amsterdump";
|
||||
srcs = [
|
||||
./main.go
|
||||
];
|
||||
|
||||
deps = with depot.third_party; [
|
||||
# gopkgs."golang.org".x.oauth2.google
|
||||
gopkgs."googlemaps.github.io".maps
|
||||
];
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,108 +0,0 @@
|
|||
// Amsterdump is a small program that populates a BigQuery table with
|
||||
// a matrix of origin points scattered around Amsterdam and each
|
||||
// respective points travel time to a given destination.
|
||||
//
|
||||
// The two destinations used here are the Schiphol Airport and
|
||||
// Amsterdam Central station.
|
||||
//
|
||||
// To accomplish this the Google Maps Distance Matrix API [1] is
|
||||
// queried with the points. A visualisation is later done using
|
||||
// BigQuery GeoViz[2].
|
||||
//
|
||||
// [1]: https://developers.google.com/maps/documentation/distance-matrix/start#quotas
|
||||
// [2]: https://bigquerygeoviz.appspot.com/
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"googlemaps.github.io/maps"
|
||||
)
|
||||
|
||||
func failOn(err error, msg string) {
|
||||
if err != nil {
|
||||
log.Fatalln(msg, err)
|
||||
}
|
||||
}
|
||||
|
||||
type LocationResult struct {
|
||||
Address string `json:"address"`
|
||||
URL string `json:"url"`
|
||||
Schiphol *maps.DistanceMatrixElement `json:"schiphol"`
|
||||
Centraal *maps.DistanceMatrixElement `json:"centraal"`
|
||||
}
|
||||
|
||||
type Listing struct {
|
||||
URL string `json:"url"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
func requestMatrix(ctx context.Context, client *maps.Client, listings []Listing) {
|
||||
origins := make([]string, len(listings))
|
||||
for i, l := range listings {
|
||||
origins[i] = l.Address
|
||||
}
|
||||
|
||||
request := &maps.DistanceMatrixRequest{
|
||||
Mode: maps.TravelModeTransit,
|
||||
Units: maps.UnitsMetric,
|
||||
Origins: origins,
|
||||
|
||||
Destinations: []string{
|
||||
"Schiphol Airport",
|
||||
"Amsterdam Centraal",
|
||||
},
|
||||
}
|
||||
|
||||
response, err := client.DistanceMatrix(ctx, request)
|
||||
failOn(err, "could not retrieve distance matrix:")
|
||||
|
||||
for idx, addr := range response.OriginAddresses {
|
||||
result := LocationResult{
|
||||
Address: addr,
|
||||
URL: listings[idx].URL,
|
||||
Schiphol: response.Rows[idx].Elements[0],
|
||||
Centraal: response.Rows[idx].Elements[1],
|
||||
}
|
||||
|
||||
j, _ := json.Marshal(result)
|
||||
fmt.Println(string(j))
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
var listings []Listing
|
||||
input, err := ioutil.ReadFile("fun/amsterdump/input.json")
|
||||
failOn(err, "could not read input file:")
|
||||
|
||||
err = json.Unmarshal(input, &listings)
|
||||
failOn(err, "could not deserialise listings:")
|
||||
|
||||
ctx := context.Background()
|
||||
apiKey := os.Getenv("MAPS_API_KEY")
|
||||
if apiKey == "" {
|
||||
log.Fatalln("API key must be supplied via MAPS_API_KEY")
|
||||
}
|
||||
|
||||
client, err := maps.NewClient(maps.WithAPIKey(apiKey))
|
||||
failOn(err, "could not create Google Maps API client:")
|
||||
|
||||
var chunk []Listing
|
||||
for _, l := range listings {
|
||||
if len(chunk) == 25 {
|
||||
requestMatrix(ctx, client, chunk)
|
||||
chunk = []Listing{}
|
||||
} else {
|
||||
chunk = append(chunk, l)
|
||||
}
|
||||
}
|
||||
|
||||
if len(chunk) > 1 {
|
||||
requestMatrix(ctx, client, chunk)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
;; Scraping funda.nl (this file is just notes and snippets, not full code)
|
||||
;;
|
||||
;; Begin by copying whole page into buffer (out of inspect element
|
||||
;; because encoding is difficult)
|
||||
|
||||
(beginning-of-buffer)
|
||||
|
||||
;; zap everything that isn't a relevant result
|
||||
(keep-lines "data-object-url-tracking\\|img alt")
|
||||
|
||||
;; mark all spans, move them to the end of the buffer
|
||||
(cl-letf (((symbol-function 'read-regexp)
|
||||
(lambda (&rest _) "</span>")))
|
||||
(mc/mark-all-in-region-regexp (point-min) (point-max)))
|
||||
|
||||
;; mark all images lines (these contain street addresses for things
|
||||
;; with images), clear up and join with previous
|
||||
;;
|
||||
;; mark all: data-image-error-fallback
|
||||
|
||||
;; delete all lines that don't either contain a span or an img tag
|
||||
;; (there are duplicates)
|
||||
(keep-lines "span class\\|img alt")
|
||||
|
||||
;; do some manual cleanup from the hrefs and done
|
||||
Loading…
Reference in a new issue