cst1/openstreetmap-website
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'pull/2486'

Browse source
This commit is contained in:
Andy Allan 2021-02-03 14:22:16 +00:00
commit 5e307c1ceb
9 changed files with 2 additions and 255 deletions
Download patch file Download diff file Expand all files Collapse all files

4
INSTALL.md
View file

@ -240,7 +240,7 @@ After installing this software, you may need to carry out some [configuration st
# Installing compiled shared library database functions (optional) # Installing compiled shared library database functions (optional)
There are special database functions required by a (little-used) API call, the migrations and diff replication. The former two are provided as *either* pure SQL functions or a compiled shared library. The SQL versions are installed as part of the recommended install procedure above and the shared library versions are recommended only if you are running a production server making a lot of `/changes` API calls or need the diff replication functionality. There are special database functions required by a (little-used) API call, the migrations and diff replication. The former two are provided as *either* pure SQL functions or a compiled shared library. The SQL versions are installed as part of the recommended install procedure above and the shared library versions are recommended only if you are running a production server and need the diff replication functionality.
If you aren't sure which you need, stick with the SQL versions. If you aren't sure which you need, stick with the SQL versions.
@ -267,14 +267,12 @@ cd ../..
If you previously installed the SQL versions of these functions, we'll need to delete those before adding the new ones: If you previously installed the SQL versions of these functions, we'll need to delete those before adding the new ones:
``` ```
psql -d openstreetmap -c "DROP FUNCTION IF EXISTS maptile_for_point"
psql -d openstreetmap -c "DROP FUNCTION IF EXISTS tile_for_point" psql -d openstreetmap -c "DROP FUNCTION IF EXISTS tile_for_point"
``` ```
Then we create the functions within each database. We're using `pwd` to substitute in the current working directory, since PostgreSQL needs the full path. Then we create the functions within each database. We're using `pwd` to substitute in the current working directory, since PostgreSQL needs the full path.
``` ```
psql -d openstreetmap -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT"
psql -d openstreetmap -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '`pwd`/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT" psql -d openstreetmap -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '`pwd`/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT"
psql -d openstreetmap -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" psql -d openstreetmap -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT"
``` ```

54
app/controllers/api/changes_controller.rb
View file

@ -1,54 +0,0 @@
module Api
class ChangesController < ApiController
authorize_resource :class => false
before_action :check_api_readable
around_action :api_call_handle_error, :api_call_timeout
# Get a list of the tiles that have changed within a specified time
# period
def index
zoom = (params[:zoom] || "12").to_i
if params.include?(:start) && params.include?(:end)
starttime = Time.parse(params[:start])
endtime = Time.parse(params[:end])
else
hours = (params[:hours] || "1").to_i.hours
endtime = Time.now.getutc
starttime = endtime - hours
end
if zoom >= 1 && zoom <= 16 &&
endtime > starttime && endtime - starttime <= 24.hours
mask = (1 << zoom) - 1
tiles = Node.where(:timestamp => starttime..endtime).group("maptile_for_point(latitude, longitude, #{zoom})").count
doc = OSM::API.new.get_xml_doc
changes = XML::Node.new "changes"
changes["starttime"] = starttime.xmlschema
changes["endtime"] = endtime.xmlschema
tiles.each do |tile, count|
x = (tile.to_i >> zoom) & mask
y = tile.to_i & mask
t = XML::Node.new "tile"
t["x"] = x.to_s
t["y"] = y.to_s
t["z"] = zoom.to_s
t["changes"] = count.to_s
changes << t
end
doc.root << changes
render :xml => doc.to_s
else
render :plain => "Requested zoom is invalid, or the supplied start is after the end time, or the start duration is more than 24 hours", :status => :bad_request
end
end
end
end

2
config/routes.rb
View file

@ -59,8 +59,6 @@ OpenStreetMap::Application.routes.draw do
get "trackpoints" => "api/tracepoints#index" get "trackpoints" => "api/tracepoints#index"
get "changes" => "api/changes#index"
get "user/:id" => "api/users#show", :id => /\d+/, :as => :api_user get "user/:id" => "api/users#show", :id => /\d+/, :as => :api_user
get "user/details" => "api/users#details" get "user/details" => "api/users#details"
get "user/gpx_files" => "api/users#gpx_files" get "user/gpx_files" => "api/users#gpx_files"

2
db/functions/Makefile
View file

@ -16,7 +16,7 @@ all: ${DESTDIR}/libpgosm.so
clean: clean:
$(RM) ${DESTDIR}/*.so ${DESTDIR}/*.o $(RM) ${DESTDIR}/*.so ${DESTDIR}/*.o
${DESTDIR}/libpgosm.so: ${DESTDIR}/quadtile.o ${DESTDIR}/maptile.o ${DESTDIR}/xid_to_int4.o ${DESTDIR}/libpgosm.so: ${DESTDIR}/quadtile.o ${DESTDIR}/xid_to_int4.o
cc ${LDFLAGS} -o $@ $^ cc ${LDFLAGS} -o $@ $^
${DESTDIR}/%.o: %.c ${DESTDIR}/%.o: %.c

27
db/functions/functions.sql
View file

@ -42,33 +42,6 @@ END;
$$ LANGUAGE plpgsql IMMUTABLE; $$ LANGUAGE plpgsql IMMUTABLE;
-- maptile_for_point returns an integer representing the tile at the given zoom
-- which contains the point (scaled_lon, scaled_lat). Note that the arguments
-- are in the order (lat, lon), and should be scaled by 10^7.
--
-- The maptile_for_point function is used only for grouping the results of the
-- (deprecated?) /changes API call. Please don't use it for anything else, as
-- it might go away in the future.
CREATE OR REPLACE FUNCTION maptile_for_point(scaled_lat int8, scaled_lon int8, zoom int4)
RETURNS int4
AS $$
DECLARE
lat CONSTANT DOUBLE PRECISION := scaled_lat / 10000000.0;
lon CONSTANT DOUBLE PRECISION := scaled_lon / 10000000.0;
zscale CONSTANT DOUBLE PRECISION := 2.0 ^ zoom;
pi CONSTANT DOUBLE PRECISION := 3.141592653589793;
r_per_d CONSTANT DOUBLE PRECISION := pi / 180.0;
x int4;
y int4;
BEGIN
-- straight port of the C code. see db/functions/maptile.c
x := floor((lon + 180.0) * zscale / 360.0);
y := floor((1.0 - ln(tan(lat * r_per_d) + 1.0 / cos(lat * r_per_d)) / pi) * zscale / 2.0);
RETURN (x << zoom) | y;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
-- xid_to_int4 converts a PostgreSQL transaction ID (xid) to a 32-bit integer -- xid_to_int4 converts a PostgreSQL transaction ID (xid) to a 32-bit integer
-- which can then be used to efficiently find rows which have changed between -- which can then be used to efficiently find rows which have changed between
-- two given transactions. This is currently used by Osmosis to extract a -- two given transactions. This is currently used by Osmosis to extract a

36
db/functions/maptile.c
View file

@ -1,36 +0,0 @@
#include <math.h>
#include <postgres.h>
#include <fmgr.h>
Datum
maptile_for_point(PG_FUNCTION_ARGS)
{
double lat = PG_GETARG_INT64(0) / 10000000.0;
double lon = PG_GETARG_INT64(1) / 10000000.0;
int zoom = PG_GETARG_INT32(2);
double scale = pow(2, zoom);
double r_per_d = M_PI / 180;
unsigned int x;
unsigned int y;
x = floor((lon + 180.0) * scale / 360.0);
y = floor((1 - log(tan(lat * r_per_d) + 1.0 / cos(lat * r_per_d)) / M_PI) * scale / 2.0);
PG_RETURN_INT32((x << zoom) | y);
}
PG_FUNCTION_INFO_V1(maptile_for_point);
/*
* To bind this into PGSQL, try something like:
*
* CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4
* AS '/path/to/rails-port/db/functions/libpgosm', 'maptile_for_point'
* LANGUAGE C STRICT;
*
* (without all the *s)
*/
#ifdef PG_MODULE_MAGIC
PG_MODULE_MAGIC;
#endif

25
db/structure.sql
View file

@ -128,31 +128,6 @@ CREATE TYPE public.user_status_enum AS ENUM (
); );
--
-- Name: maptile_for_point(bigint, bigint, integer); Type: FUNCTION; Schema: public; Owner: -
--
CREATE FUNCTION public.maptile_for_point(scaled_lat bigint, scaled_lon bigint, zoom integer) RETURNS integer
LANGUAGE plpgsql IMMUTABLE
AS $$
DECLARE
lat CONSTANT DOUBLE PRECISION := scaled_lat / 10000000.0;
lon CONSTANT DOUBLE PRECISION := scaled_lon / 10000000.0;
zscale CONSTANT DOUBLE PRECISION := 2.0 ^ zoom;
pi CONSTANT DOUBLE PRECISION := 3.141592653589793;
r_per_d CONSTANT DOUBLE PRECISION := pi / 180.0;
x int4;
y int4;
BEGIN
-- straight port of the C code. see db/functions/maptile.c
x := floor((lon + 180.0) * zscale / 360.0);
y := floor((1.0 - ln(tan(lat * r_per_d) + 1.0 / cos(lat * r_per_d)) / pi) * zscale / 2.0);
RETURN (x << zoom) | y;
END;
$$;
-- --
-- Name: tile_for_point(integer, integer); Type: FUNCTION; Schema: public; Owner: - -- Name: tile_for_point(integer, integer); Type: FUNCTION; Schema: public; Owner: -
-- --

1
script/vagrant/setup/provision.sh
View file

@ -52,7 +52,6 @@ sudo -u vagrant psql -d openstreetmap -f db/functions/functions.sql
################################################################################ ################################################################################
#pushd db/functions #pushd db/functions
#sudo -u vagrant make #sudo -u vagrant make
#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'maptile_for_point' LANGUAGE C ST#RICT"
#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'tile_for_point' LANGUAGE C STRICT" #sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'tile_for_point' LANGUAGE C STRICT"
#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'xid_to_int4' LANGUAGE C STRICT" #sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'xid_to_int4' LANGUAGE C STRICT"
#popd #popd

106
test/controllers/api/changes_controller_test.rb
View file

@ -1,106 +0,0 @@
require "test_helper"
module Api
class ChangesControllerTest < ActionDispatch::IntegrationTest
##
# test all routes which lead to this controller
def test_routes
assert_routing(
{ :path => "/api/0.6/changes", :method => :get },
{ :controller => "api/changes", :action => "index" }
)
end
# MySQL and Postgres require that the C based functions are installed for
# this test to work. More information is available from:
# http://wiki.openstreetmap.org/wiki/Rails#Installing_the_quadtile_functions
# or by looking at the readme in db/README
def test_changes_simple
# create a selection of nodes
(1..5).each do |n|
create(:node, :timestamp => Time.utc(2007, 1, 1, 0, 0, 0), :lat => n, :lon => n)
end
# deleted nodes should also be counted
create(:node, :deleted, :timestamp => Time.utc(2007, 1, 1, 0, 0, 0), :lat => 6, :lon => 6)
# nodes in the same tile won't change the total
create(:node, :timestamp => Time.utc(2007, 1, 1, 0, 0, 0), :lat => 6, :lon => 6)
# nodes with a different timestamp should be ignored
create(:node, :timestamp => Time.utc(2008, 1, 1, 0, 0, 0), :lat => 7, :lon => 7)
travel_to Time.utc(2010, 4, 3, 10, 55, 0) do
get changes_path
assert_response :success
now = Time.now.getutc
hourago = now - 1.hour
assert_select "osm[version='#{Settings.api_version}'][generator='#{Settings.generator}']", :count => 1 do
assert_select "changes[starttime='#{hourago.xmlschema}'][endtime='#{now.xmlschema}']", :count => 1 do
assert_select "tile", :count => 0
end
end
end
travel_to Time.utc(2007, 1, 1, 0, 30, 0) do
get changes_path
assert_response :success
# print @response.body
# As we have loaded the fixtures, we can assume that there are some
# changes at the time we have frozen at
now = Time.now.getutc
hourago = now - 1.hour
assert_select "osm[version='#{Settings.api_version}'][generator='#{Settings.generator}']", :count => 1 do
assert_select "changes[starttime='#{hourago.xmlschema}'][endtime='#{now.xmlschema}']", :count => 1 do
assert_select "tile", :count => 6
end
end
end
end
def test_changes_zoom_invalid
zoom_to_test = %w[p -1 0 17 one two]
zoom_to_test.each do |zoom|
get changes_path(:zoom => zoom)
assert_response :bad_request
assert_equal("Requested zoom is invalid, or the supplied start is after the end time, or the start duration is more than 24 hours", @response.body)
end
end
def test_changes_zoom_valid
1.upto(16) do |zoom|
get changes_path(:zoom => zoom)
assert_response :success
# NOTE: there was a test here for the timing, but it was too sensitive to be a good test
# and it was annoying.
assert_select "osm[version='#{Settings.api_version}'][generator='#{Settings.generator}']", :count => 1 do
assert_select "changes", :count => 1
end
end
end
def test_changes_hours_invalid
invalid = %w[-21 335 -1 0 25 26 100 one two three ping pong :]
invalid.each do |hour|
get changes_path(:hours => hour)
assert_response :bad_request, "Problem with the hour: #{hour}"
assert_equal("Requested zoom is invalid, or the supplied start is after the end time, or the start duration is more than 24 hours", @response.body, "Problem with the hour: #{hour}.")
end
end
def test_changes_hours_valid
1.upto(24) do |hour|
get changes_path(:hours => hour)
assert_response :success
end
end
def test_changes_start_end_invalid
get changes_path(:start => "2010-04-03 10:55:00", :end => "2010-04-03 09:55:00")
assert_response :bad_request
assert_equal("Requested zoom is invalid, or the supplied start is after the end time, or the start duration is more than 24 hours", @response.body)
end
def test_changes_start_end_valid
get changes_path(:start => "2010-04-03 09:55:00", :end => "2010-04-03 10:55:00")
assert_response :success
end
end
end