json_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying gs://location_history_bucket/rawLocationHistory_20190623.json...\n",
      "/ [0 files][    0.0 B/448.0 MiB]                                                \r",
      "==> NOTE: You are downloading one or more large file(s), which would\n",
      "run significantly faster if you enabled sliced object downloads. This\n",
      "feature is enabled by default but requires that compiled crcmod be\n",
      "installed (see \"gsutil help crcmod\").\n",
      "\n",
      "-\r",
      "- [0 files][ 69.4 MiB/448.0 MiB]                                                \r",
      "\\\r",
      "|\r",
      "| [0 files][219.4 MiB/448.0 MiB]                                                \r",
      "/\r",
      "/ [0 files][366.4 MiB/448.0 MiB]                                                \r",
      "-\r",
      "- [1 files][448.0 MiB/448.0 MiB]                                                \r",
      "\\\r\n",
      "Operation completed over 1 objects/448.0 MiB.                                    \n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "gsutil cp \"gs://location_history_bucket/rawLocationHistory_20190623.json\" ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./rawLocationHistory_20190623.json') as json_file:\n",
    "    data = json.load(json_file)\n",
    "\n",
    "output_filename = './processLocationHistory_20190623.json'\n",
    "\n",
    "open(output_filename, 'w').close()\n",
    "\n",
    "with open(output_filename, 'a') as outfile:\n",
    "    for stamp in data['locations']:\n",
    "        outfile.write(str(stamp) + os.linesep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying file://./processLocationHistory_20190623.json [Content-Type=application/json]...\n",
      "/ [0 files][    0.0 B/317.4 MiB]                                                \r",
      "==> NOTE: You are uploading one or more large file(s), which would run\n",
      "significantly faster if you enable parallel composite uploads. This\n",
      "feature can be enabled by editing the\n",
      "\"parallel_composite_upload_threshold\" value in your .boto\n",
      "configuration file. However, note that if you do this large files will\n",
      "be uploaded as `composite objects\n",
      "<https://cloud.google.com/storage/docs/composite-objects>`_,which\n",
      "means that any user who downloads such objects will need to have a\n",
      "compiled crcmod installed (see \"gsutil help crcmod\"). This is because\n",
      "without a compiled crcmod, computing checksums on composite objects is\n",
      "so slow that gsutil disables downloads of composite objects.\n",
      "\n",
      "-\r",
      "- [0 files][131.2 MiB/317.4 MiB]                                                \r",
      "\\\r",
      "|\r",
      "| [0 files][297.3 MiB/317.4 MiB]                                                \r",
      "| [1 files][317.4 MiB/317.4 MiB]                                                \r",
      "/\r\n",
      "Operation completed over 1 objects/317.4 MiB.                                    \n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "gsutil cp \"./processLocationHistory_20190623.json\" \"gs://location_history_bucket/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}