1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
# SPDX-FileCopyrightText: 2020 IN COMMON Collective <collective@incommon.cc>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
#This file parses the data from the "Concertes" and "Dewey" folders
#It creates a file "Locations.geojson" which is in the geoJson format to be used with a map
require 'json'
require 'csv'
class Parser
def initialize()
@list_of_locations = {} #Contains all the locations from the Dewey and Concertes data
@list_of_locations["type"] = "FeatureCollection"
@list_of_locations["features"] = []
end
def sanitize(data)
#Sanitize data. Puts "" instead of null in empty data, and puts "" instead of "/"
if (data == nil)
data = ""
end
data = data.strip
res = data
if (data == "/")
res = ""
end
return res
end
def parseDeweyFiles(data_file)
#Parses the .json files contained in the Dewey folder
data = JSON.parse(File.read(data_file))
i = 0
while i < data.size
new_item = {}
new_item["type"] = "Feature"
new_item["geometry"] = {}
new_item["geometry"]["type"] = "Point"
coords = data[i]["fields"]["position"].split(";")[1].gsub("POINT (","").gsub(")","").split(" ")
coords[0] = coords[0].to_f
coords[1] = coords[1].to_f
new_item["geometry"]["coordinates"] = coords
new_item["properties"] = {}
new_item["properties"]["name"] = data[i]["fields"]["name"]
new_item["properties"]["description"] = data[i]["fields"]["comment"]
new_item["properties"]["entry_number"] = data[i]["pk"]
new_item["properties"]["phone_number"] = sanitize(data[i]["fields"]["phone"])
new_item["properties"]["website"] = sanitize(data[i]["fields"]["web"])
new_item["properties"]["email"] = sanitize(data[i]["fields"]["email"])
new_item["properties"]["address"] = data[i]["fields"]["address"]
new_item["properties"]["city"] = "" #No equivalent in Dewey data
new_item["properties"]["postal_code"] = "" #No equivalent in Dewey data
new_item["properties"]["categories"] = data[i]["fields"]["subcategories"]
new_item["properties"]["source"] = "Dewey"
new_item["properties"]["srid"] = data[i]["fields"]["position"].split(";")[0].gsub("SRID=","")
@list_of_locations["features"].push(new_item)
i += 1
end
end
def parseConcertesFiles(data_file, coordinates_file)
#Parses .csv files contained in the Concertes folder
data = CSV.parse(File.read(data_file), headers: true, :col_sep => ";")
coord = CSV.parse(File.read(coordinates_file), headers: true, :col_sep => ";")
i = 0
while i < data.size
new_item = {}
new_item["type"] = "Feature"
new_item["geometry"] = {}
new_item["geometry"]["type"] = "Point"
new_item["geometry"]["coordinates"] = [coord[i]["CoordLamb_X"].to_f, coord[i]["CoordLamb_Y"].to_f]
new_item["properties"] = {}
new_item["properties"]["name"] = data[i]["Denomination_FULL"]
new_item["properties"]["description"] = data[i]["description"]
new_item["properties"]["entry_number"] = data[i]["NumEntr"]
new_item["properties"]["phone_number"] = sanitize(coord[i]["Tel"])
new_item["properties"]["website"] = sanitize(coord[i]["Web"])
new_item["properties"]["email"] = sanitize(coord[i]["Email"])
new_item["properties"]["address"] = coord[i]["Adresse"]
new_item["properties"]["city"] = data[i]["INS_COMMUNE"]
new_item["properties"]["postal_code"] = coord[i]["Code postal"]
new_item["properties"]["categories"] = [] #The categories from the Concertes data will not be used
new_item["properties"]["source"] = "Concertes"
new_item["properties"]["srid"] = "4326" #Not precised in Concertes data
@list_of_locations["features"].push(new_item)
i += 1
end
end
def writeToFile(write_file)
#Writes the list of locations in a pretty Json file
File.write(write_file, JSON.pretty_generate(@list_of_locations))
end
end
BEGIN {
puts "Starting parser"
}
parser = Parser.new()
parser.parseConcertesFiles("Concertes/20200312_EntreprisesSignaletique.csv", "Concertes/20200312_EntreprisesCoordonnees.csv")
parser.parseDeweyFiles("Dewey/dewey-maps-markers.json")
parser.writeToFile("Locations.geojson")
END {
puts "Ending parser, the data is available in the file 'Locations.geoJson'"
}
|
