Merge branch 'master' into openstreetbugs

Conflicts:
	Gemfile.lock
	app/views/browse/_map.html.erb
	app/views/user/view.html.erb
	config/locales/en.yml
	config/openlayers.cfg
	db/structure.sql
	vendor/assets/openlayers/OpenLayers.js

lib/not_redactable.rb

@@ -0,0 +1,11 @@
+require 'osm'
+
+module NotRedactable
+  def redacted?
+    false
+  end
+
+  def redact!(r)
+    raise OSM::APICannotRedactError.new
+  end
+end

lib/osm.rb

@@ -6,7 +6,6 @@ module OSM
   require 'rexml/text'
   require 'xml/libxml'
   require 'digest/md5'
-  require 'nokogiri'
 
   if defined?(SystemTimer)
     Timer = SystemTimer
@@ -269,6 +268,19 @@ module OSM
     end
   end
 
+  ##
+  # raised when someone tries to redact a current version of
+  # an element - only historical versions can be redacted.
+  class APICannotRedactError < APIError
+    def status
+      :bad_request
+    end
+
+    def to_s
+      "Cannot redact current version of element, only historical versions may be redacted."
+    end
+  end
+
   # Helper methods for going to/from mercator and lat/lng.
   class Mercator
     include Math
@@ -511,27 +523,6 @@ module OSM
                       "AND #{prefix}longitude BETWEEN #{bbox.min_lon} AND #{bbox.max_lon}"
   end
 
-  # Return a spam score for a chunk of text
-  def self.spam_score(text)
-    link_count = 0
-    link_size = 0
-
-    doc = Nokogiri::HTML(Rinku.auto_link(text, :urls))
-
-    if doc.content.length > 0
-      doc.xpath("//a").each do |link|
-        link_count += 1
-        link_size += link.content.length
-      end
-
-      link_proportion = link_size.to_f / doc.content.length.to_f
-    else
-      link_proportion = 0
-    end
-
-    return [link_proportion - 0.2, 0.0].max * 200 + link_count * 20
-  end
-
   def self.legal_text_for_country(country_code)
     file_name = File.join(Rails.root, "config", "legales", country_code.to_s + ".yml")
     file_name = File.join(Rails.root, "config", "legales", DEFAULT_LEGALE + ".yml") unless File.exist? file_name

lib/potlatch.rb

@@ -8,18 +8,20 @@ module Potlatch
     
     # Return two-byte integer
     def self.getint(s) 
-      s.getc*256+s.getc
+      s.getbyte*256+s.getbyte
     end
 
     # Return four-byte long
     def self.getlong(s) 
-      ((s.getc*256+s.getc)*256+s.getc)*256+s.getc
+      ((s.getbyte*256+s.getbyte)*256+s.getbyte)*256+s.getbyte
     end
 
     # Return string with two-byte length 
     def self.getstring(s) 
-      len=s.getc*256+s.getc
-      s.read(len)
+      len=s.getbyte*256+s.getbyte
+      str=s.read(len)
+      str.force_encoding("UTF-8") if str.respond_to?("force_encoding")
+      str
     end
 
     # Return eight-byte double-precision float 
@@ -45,23 +47,23 @@ module Potlatch
         if (key=='') then break end
         arr[key]=getvalue(s)
       end
-      s.getc		# skip the 9 'end of object' value
+      s.getbyte		# skip the 9 'end of object' value
       arr
     end
 
     # Parse and get value
     def self.getvalue(s) 
-      case s.getc
+      case s.getbyte
       when 0;	return getdouble(s)			# number
-      when 1;	return s.getc				# boolean
+      when 1;	return s.getbyte			# boolean
       when 2;	return getstring(s)			# string
       when 3;	return getobject(s)			# object/hash
-      when 5;	return nil					# null
-      when 6;	return nil					# undefined
-      when 8;	s.read(4)					# mixedArray
-        return getobject(s)			#  |
-      when 10;return getarray(s)			# array
-      else;	return nil					# error
+      when 5;	return nil				# null
+      when 6;	return nil				# undefined
+      when 8;	s.read(4)				# mixedArray
+                return getobject(s)			#  |
+      when 10;  return getarray(s)			# array
+      else;	return nil				# error
       end
     end
 
@@ -134,7 +136,7 @@ module Potlatch
       # Skip headers
       AMF.getint(@request).times do     # Read number of headers and loop
         AMF.getstring(@request)         #  | skip name
-        req.getc                        #  | skip boolean
+        req.getbyte                     #  | skip boolean
         AMF.getvalue(@request)          #  | skip value
       end
 

lib/potlatch2.rb

@@ -1,16 +1,36 @@
 module Potlatch2
   LOCALES = {
+    "ar" => "ar",
     "ast" => "ast",
+    "be" => "be-tarask",
     "br" => "br",
+    "ca" => "ca",
+    "cs" => "cs_CZ",
+    "da" => "da",
     "de" => "de_DE",
+    "diq" => "diq",
+    "dsb" => "dsb",
     "en" => "en_US",
     "en-GB" => "en_GB",
+    "el" => "el",
     "es" => "es_ES",
+    "et" => "et",
+    "fa" => "fa",
     "fi" => "fi",
     "fr" => "fr_FR",
+    "ga" => "ga",
+    "gl" => "gl",
     "he" => "he",
     "hsb" => "hsb",
+    "hu" => "hu",
+    "ia" => "ia",
+    "it" => "it_IT",
     "ja" => "ja_JP",
+    "ka" => "ka",
+    "ko" => "ko",
+    "lb" => "lb",
+    "lt" => "lt",
+    "lv" => "lv",
     "mk" => "mk",
     "ms" => "ms",
     "nb" => "nb_NO",
@@ -18,9 +38,20 @@ module Potlatch2
     "nn" => "nn_NO",
     "no" => "nb_NO",
     "pl" => "pl_PL",
+    "pt" => "pt_PT",
+    "pt-BR" => "pt_BR",
     "ru" => "ru",
+    "rue" => "rue",
+    "sk" => "sk",
+    "sl" => "sl",
     "sq" => "sq",
+    "sr-EC" => "sr-ec",
+    "sr-EL" => "sr-el",
+    "sv" => "sv_SE",
+    "ta" => "ta",
+    "tl" => "tl",
     "tr" => "tr",
+    "uk" => "uk",
     "vi" => "vi_VN",
     "zh" => "zh_CN"
   }

lib/redactable.rb

@@ -0,0 +1,22 @@
+require 'osm'
+
+module Redactable
+  def self.included(base)
+    # this is used to extend activerecord bases, as these aren't
+    # in scope for the module itself.
+    base.scope :unredacted, base.where(:redaction_id => nil)
+  end
+  
+  def redacted?
+    not self.redaction.nil?
+  end
+
+  def redact!(redaction)
+    # check that this version isn't the current version
+    raise OSM::APICannotRedactError.new if self.is_latest_version?
+
+    # make the change
+    self.redaction = redaction
+    self.save!
+  end
+end

lib/rich_text.rb

@@ -0,0 +1,117 @@
+module RichText
+  def self.new(format, text)
+    case format
+    when "html"; HTML.new(text || "")
+    when "markdown"; Markdown.new(text || "")
+    when "text"; Text.new(text || "")
+    else; nil
+    end
+  end
+
+  class SimpleFormat
+    include ActionView::Helpers::TextHelper
+
+    def sanitize(text)
+      Sanitize.clean(text, Sanitize::Config::OSM).html_safe
+    end
+  end
+
+  class Base < String
+    include ActionView::Helpers::TagHelper
+
+    def spam_score
+      link_count = 0
+      link_size = 0
+
+      doc = Nokogiri::HTML(to_html)
+
+      if doc.content.length > 0
+        doc.xpath("//a").each do |link|
+          link_count += 1
+          link_size += link.content.length
+        end
+
+        link_proportion = link_size.to_f / doc.content.length.to_f
+      else
+        link_proportion = 0
+      end
+
+      return [link_proportion - 0.2, 0.0].max * 200 + link_count * 40
+    end
+
+  protected
+
+    def simple_format(text)
+      SimpleFormat.new.simple_format(text)
+    end
+
+    def linkify(text)
+      if text.html_safe?
+        Rinku.auto_link(text, :urls, tag_options(:rel => "nofollow")).html_safe
+      else
+        Rinku.auto_link(text, :urls, tag_options(:rel => "nofollow"))
+      end
+    end
+  end
+
+  class HTML < Base
+    def to_html
+      linkify(sanitize(simple_format(self)))
+    end
+
+    def to_text
+      self.to_s
+    end
+
+  private
+
+    def sanitize(text)
+      Sanitize.clean(text, Sanitize::Config::OSM).html_safe
+    end
+  end
+
+  class Markdown < Base
+    def to_html
+      html_parser.render(self).html_safe
+    end
+
+    def to_text
+      self.to_s
+    end
+
+  private
+
+    def html_parser
+      @@html_renderer ||= Renderer.new({
+        :filter_html => true, :safe_links_only => true
+      })
+      @@html_parser ||= Redcarpet::Markdown.new(@@html_renderer, {
+        :no_intra_emphasis => true, :autolink => true, :space_after_headers => true
+      })
+    end
+
+    class Renderer < Redcarpet::Render::XHTML
+      def link(link, title, alt_text)
+        "<a rel=\"nofollow\" href=\"#{link}\">#{alt_text}</a>"
+      end
+
+      def autolink(link, link_type)
+        if link_type == :email
+          "<a rel=\"nofollow\" href=\"mailto:#{link}\">#{link}</a>"
+        else
+          "<a rel=\"nofollow\" href=\"#{link}\">#{link}</a>"
+        end
+      end 
+    end
+  end
+
+  class Text < Base
+    def to_html
+      linkify(simple_format(ERB::Util.html_escape(self)))
+    end
+
+    def to_text
+      self.to_s
+    end
+  end
+end

lib/utf8.rb

@@ -1,16 +1,21 @@
-require 'iconv'
-
 module UTF8
   ##
   # Checks that a string is valid UTF-8 by trying to convert it to UTF-8
   # using the iconv library, which is in the standard library.
-  def self.valid?(str)
-    return true if str.nil?
-    Iconv.conv("UTF-8", "UTF-8", str)
-    return true
-    
-  rescue
-    return false
-  end  
-end
+  if String.new.respond_to?("valid_encoding?")
+    def self.valid?(str)
+      return true if str.nil?
+      return str.valid_encoding?
+    end
+  else
+    require 'iconv'
 
+    def self.valid?(str)
+      return true if str.nil?
+      Iconv.conv("UTF-8", "UTF-8", str)
+      return true
+    rescue
+      return false
+    end
+  end
+end