Files to parse with parser.rb to create the Locations.geojson file which contains all the locations from the Dewey and Concertes data

1 files changed, 101 insertions, 0 deletions

diff --git a/doc/import/parser.rb b/doc/import/parser.rb
new file mode 100644
index 0000000..1c5d217
--- /dev/null
+++ b/doc/import/parser.rb
@@ -0,0 +1,101 @@
+require 'json'
+require 'csv'
+require 'set'
+
+class Parser
+	def initialize()
+		@list_of_locations = {}
+		@list_of_locations["type"] = "FeatureCollection"
+		@list_of_locations["features"] = []
+		
+	end
+	def parseDeweyFiles(data_file)
+		data = JSON.parse(File.read(data_file))
+		i = 0
+		cat_nbr = 0
+		while i < data.size
+			new_item = {}
+			new_item["type"] = "Feature"
+			new_item["geometry"] = {}
+			new_item["geometry"]["type"] = "Point"
+			coords = data[i]["fields"]["position"].split(";")[1].gsub("POINT (","").gsub(")","").split(" ")
+			coords[0] = coords[0].to_f
+			coords[1] = coords[1].to_f
+			new_item["geometry"]["coordinates"] = coords
+			new_item["properties"] = {}
+			new_item["properties"]["name"] = data[i]["fields"]["name"]
+			new_item["properties"]["description"] = data[i]["fields"]["comment"]
+			new_item["properties"]["entry_number"] = data[i]["pk"]
+			new_item["properties"]["phone_number"] = data[i]["fields"]["phone"]
+			new_item["properties"]["website"] = data[i]["fields"]["web"]
+			new_item["properties"]["mail"] = data[i]["fields"]["email"]
+			new_item["properties"]["address"] = data[i]["fields"]["address"]
+			new_item["properties"]["city"] = "" #No equivalent in Dewey database
+			new_item["properties"]["postal_code"] = "" #No equivalent in Dewey database
+			new_item["properties"]["categories"] = data[i]["fields"]["subcategories"]
+			#if data[i]["fields"]["subcategories"][0] == 50
+			#	cat_nbr += 1
+			#end
+			new_item["properties"]["source"] = "Dewey"
+			new_item["properties"]["srid"] = data[i]["fields"]["position"].split(";")[0].gsub("SRID=","")
+			@list_of_locations["features"].push(new_item)
+			i += 1
+		end
+		#puts cat_nbr
+	end
+	def parseConcertesFiles(data_file, coordinates_file)
+		data = CSV.parse(File.read(data_file), headers: true, :col_sep => ";")
+		coord = CSV.parse(File.read(coordinates_file), headers: true, :col_sep => ";")
+		i = 0
+		while i < data.size
+			new_item = {}
+			new_item["type"] = "Feature"
+			new_item["geometry"] = {}
+			new_item["geometry"]["type"] = "Point"
+			new_item["geometry"]["coordinates"] = [coord[i]["CoordLamb_X"].to_f, coord[i]["CoordLamb_Y"].to_f]
+			new_item["properties"] = {}
+			new_item["properties"]["name"] = data[i]["Denomination_FULL"]
+			new_item["properties"]["description"] = data[i]["description"]
+			new_item["properties"]["entry_number"] = data[i]["NumEntr"]
+			new_item["properties"]["phone_number"] = coord[i]["Tel"]
+			new_item["properties"]["website"] = coord[i]["Web"]
+			new_item["properties"]["mail"] = coord[i]["Email"]
+			new_item["properties"]["address"] = coord[i]["Adresse"]
+			new_item["properties"]["city"] = data[i]["INS_COMMUNE"]
+			new_item["properties"]["postal_code"] = coord[i]["Code postal"]
+			new_item["properties"]["categories"] = []
+			new_item["properties"]["source"] = "Concertes"
+			new_item["properties"]["srid"] = "4326" #Not precised in Concertes database
+			@list_of_locations["features"].push(new_item)
+			i += 1
+		end
+		File.write("Locations.geojson", JSON.pretty_generate(@list_of_locations))
+	end
+	def writeToFile(write_file)
+		File.write(write_file, JSON.pretty_generate(@list_of_locations))
+	end
+	def seeCategories(data_file)
+		data = CSV.parse(File.read(data_file), headers: true, :col_sep => ";")
+		i = 0
+		p = Set.new
+		while i < data.size
+			p << data[i]["Cat"]
+			i += 1
+		end
+		puts p
+	end
+end
+
+BEGIN {
+	puts "Starting parser"
+}
+parser = Parser.new()
+parser.parseConcertesFiles("Concertes/20200312_EntreprisesSignaletique.csv", "Concertes/20200312_EntreprisesCoordonnees.csv")
+parser.parseDeweyFiles("Dewey/dewey-maps-markers.json")
+parser.writeToFile("Locations.geojson")
+#parser.seeCategories("Concertes/20200312_CodeMotCle.csv")
+
+
+END {
+	puts "Ending parser"
+}