diff --git a/.gitignore b/.gitignore index 88e4944..ea644bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Tokyo GTFS gtfs/* +output/* temp/* ekikara/* ignore* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7d93d2b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 + +# create the base image with the dependencies +FROM python:3.7-alpine AS baseimage +WORKDIR /install/ +COPY requirements.txt requirements.txt +RUN pip3 install -r requirements.txt + +# Now add in the app source code +FROM baseimage AS build +WORKDIR /app/ +COPY /data ./data +COPY /src ./src +COPY *.py . +ENTRYPOINT ["python3"] diff --git a/README.md b/README.md index 6078f4c..e005e6b 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,34 @@ UPDATING TOKYO GTFS R2R PACKAGE - Add the output `trains.zip` and `buses.zip` to the [GTFS Feed Manager](https://content.rome2rio.com/gtfs-feed-manager) to `jp.tokyo.buses` and `jp.tokyo.trains` respectively and save each one - Wait until the GTFS Scraper & Transit Builder next run, and then these new files will be added to the product -Description +Docker build process ----------- +* Build the container image first + * `docker build -t r2r/tokyogtfs:latest .` +* Then run the container with the command line args and set the api key. The API key can be found in 1password or you will need to sign up here [Public Transportation Open Data Centre](https://www.odpt.org/en/) + * Double check the mounted directory matches the fulle path to the output folder that you want. It is the left hand side of the `-v` arg up to the colon `:` + * `docker run -v C:\src\TokyoGTFS\output:/app/output --name tokyogtfs r2r/tokyogtfs:latest run_gtfs.py -a YOUR-APIKEY` +* Check the output folder `output` for the zip files and add them to the JIRA ticket for a content user to perform the upload and testing + +Approximate time for completion as of 2022-12-06: +* trains - 80 seconds +* busses - 12 minutes + -Make GTFS and GTFS-Realtime feeds for Tokyo from data provided by [Open Data Challenge for Public Transportation in Tokyo](https://tokyochallenge.odpt.org/). +Original ReadMe +----------- +Description +----------- +Make GTFS and GTFS-Realtime feeds for Tokyo from data provided by [Public Transportation Open Data Centre](https://www.odpt.org/en/). Precautions ----------- Before using this script you're going to have to get an apikey for Open Data Challenge. You can do this at the [ODPT website](https://tokyochallenge.odpt.org/en/index.html#entry). -+++ API Key can be found here: https://developer-tokyochallenge.odpt.org/oauth/applications?locale=en +++ ++++ API Key can be found here: https://api.odpt.org/oauth/applications?locale=en +++ Then put this apikey in a file called `apikey.txt` where python scripts are provided, or provide it as command line arguments for the script, like `python3 .py -a YOUR-APIKEY`. +++ This Key can also be found in LastPass, under "Tokyo GTFS Key" in Shared-Content Tools @@ -59,8 +74,8 @@ Windows users may need to run `py -m pip ...` instead of `pip3 ...` and `py ...` Attributions ------------ -Use created data according to [API Use Guidelines](https://developer-tokyochallenge.odpt.org/en/terms/api_guideline.html), -[API Use Permission Rules](https://developer-tokyochallenge.odpt.org/en/terms/terms_api_usage.html). +Use created data according to [API Use Guidelines](https://api.odpt.org/en/terms/api_guideline.html), +[API Use Permission Rules](https://api.odpt.org/en/terms/terms_api_usage.html). The source of data used for GTFS-creating scripts is the Open Data Challenge for Public Transportation in Tokyo. They are based on the data provided by the public transportation operators. diff --git a/buses_gtfs.py b/buses_gtfs.py index 2612e26..dc00e4b 100644 --- a/buses_gtfs.py +++ b/buses_gtfs.py @@ -9,7 +9,7 @@ if __name__ == "__main__": args_parser = argparse.ArgumentParser() args_parser.add_argument("-a", "--apikey", metavar="YOUR_APIKEY", - help="apikey from developer-tokyochallenge.odpt.org") + help="apikey from api.odpt.org") args = args_parser.parse_args() # Apikey checks diff --git a/run_gtfs.py b/run_gtfs.py new file mode 100644 index 0000000..a42d7cc --- /dev/null +++ b/run_gtfs.py @@ -0,0 +1,50 @@ +import argparse +import time +import sys +import os + +from src.static_buses import BusParser +from src.static_trains import TrainParser +from src.const import HEADER + +if __name__ == "__main__": + args_parser = argparse.ArgumentParser() + args_parser.add_argument("-a", "--apikey", metavar="YOUR_APIKEY", + help="apikey from api.odpt.org") + args = args_parser.parse_args() + + # Apikey checks + if args.apikey: + apikey = args.apikey + + elif os.path.exists("apikey.txt"): + with open("apikey.txt", mode="r", encoding="utf8") as f: + apikey = f.read().strip() + + else: + sys.exit( + "No apikey!\n" + "Provide it inside command line argument '--apikey',\n" + "Or put it inside a file named 'apikey.txt'." + ) + + start_time = time.time() + print(HEADER) + print("=== Trains GTFS ===") + + print("Warming up") + TrainParser.parse(apikey) + + total_time = time.time() - start_time + print("=== Trains GTFS: Finished in {} s ===".format(round(total_time, 2))) + + + start_time = time.time() + print(HEADER) + print("=== Buses GTFS ===") + + print("Warming up") + BusParser.parse(apikey) + + total_time = time.time() - start_time + print("=== Buses GTFS: Finished in {} s ===".format(round(total_time, 2))) \ No newline at end of file diff --git a/src/handlers.py b/src/handlers.py index f7780ee..fbca8b9 100644 --- a/src/handlers.py +++ b/src/handlers.py @@ -118,7 +118,7 @@ def get(self, endpoint, data_dump=True, force_vanilla_json=False): print_log(f"Requesting data dump for {endpoint}") req = self.session.get( - f"https://api-tokyochallenge.odpt.org/api/v4/odpt:{endpoint}", + f"https://api.odpt.org/api/v4/odpt:{endpoint}", params={"acl:consumerKey": self.apikey}, timeout=GET_TIMEOUT, stream=True, ) diff --git a/src/utils.py b/src/utils.py index 51fe6ef..85fa82f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -84,6 +84,7 @@ def compress(target_file): :type target_file: str or file-like or path-like """ + target_file = "output/" + target_file print_log(f"Compressing to {target_file}") with zipfile.ZipFile(target_file, mode="w", compression=zipfile.ZIP_DEFLATED) as arch: diff --git a/trains_gtfs.py b/trains_gtfs.py index f71d993..3cfd34c 100644 --- a/trains_gtfs.py +++ b/trains_gtfs.py @@ -9,7 +9,7 @@ if __name__ == "__main__": args_parser = argparse.ArgumentParser() args_parser.add_argument("-a", "--apikey", metavar="YOUR_APIKEY", - help="apikey from developer-tokyochallenge.odpt.org") + help="apikey from api.odpt.org") args = args_parser.parse_args() # Apikey checks diff --git a/trains_realtime.py b/trains_realtime.py index 3639765..9d0099f 100644 --- a/trains_realtime.py +++ b/trains_realtime.py @@ -11,7 +11,7 @@ args_parser.add_argument( "-a", "--apikey", metavar="YOUR-APIKEY", - help="apikey from developer-tokyochallenge.odpt.org" + help="apikey from api.odpt.org" ) args_parser.add_argument(