<%= flash_message %>
@@ -72,6 +75,19 @@ <%= render @view_module, @view_template, assigns %>diff --git a/.dockerignore b/.dockerignore index 57ec6513..c66afacd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,5 @@ README* test/ priv/static/ **/node_modules +import/ +teslamate.bck diff --git a/.gitignore b/.gitignore index e1df6955..449fb156 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ npm-debug.log config/*.env /deploy/ +/import/ conf/wizzy.json .env @@ -54,4 +55,4 @@ conf/wizzy.json # other stuff created by my build env?! .elixir_ls/ -assets/package-lock.json \ No newline at end of file +assets/package-lock.json diff --git a/README.md b/README.md index 9c1ce183..e11a9570 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ A powerful, self-hosted data logger for your Tesla. - Geo-fencing feature to create custom locations - Supports multiple vehicles per Tesla Account - Charge cost tracking +- Import from TeslaFi ## Screenshots @@ -68,6 +69,8 @@ The full TeslaMate documentation is available on [Read the Docs](https://teslama - [Shortcuts Setup (iOS)](https://teslamate.readthedocs.io/en/latest/configuration/guides/shortcuts.html) - [Tasker Setup (Android)](https://teslamate.readthedocs.io/en/latest/configuration/guides/tasker.html) - [MacroDroid Setup (Android)](https://teslamate.readthedocs.io/en/latest/configuration/guides/macro_droid.html) +- Import + - [TeslaFi](https://teslamate.readthedocs.io/en/latest/import/teslafi.html) - Integrations - [HomeAssistant](https://teslamate.readthedocs.io/en/latest/integrations/home_assistant.html) - [MQTT](https://teslamate.readthedocs.io/en/latest/integrations/mqtt.html) diff --git a/assets/css/app.scss b/assets/css/app.scss index 746e56b5..8e27ca18 100644 --- a/assets/css/app.scss +++ b/assets/css/app.scss @@ -14,9 +14,9 @@ $fullhd-enabled: false; @import "~bulma/sass/base/_all"; @import "~bulma/sass/components/breadcrumb"; @import "~bulma/sass/components/card"; +@import "~bulma/sass/components/dropdown"; @import "~bulma/sass/components/message"; @import "~bulma/sass/components/navbar"; -@import "~bulma/sass/components/dropdown"; @import "~bulma/sass/elements/box"; @import "~bulma/sass/elements/button"; @import "~bulma/sass/elements/container"; @@ -253,3 +253,17 @@ $positions: ('top','left','bottom','right'); .field.is-horizontal .field-label { min-width: 200px; } + +main.full-height { + min-height: calc(100vh - 102px); + min-height: calc((var(--vh, 1vh) * 100) - 102px); +} + +.footer { + height: 50px; + opacity: 0.8; + + .button.is-text { + text-decoration: initial; + } +} diff --git a/assets/js/main.js b/assets/js/main.js index 31bca6a7..2a44b76e 100644 --- a/assets/js/main.js +++ b/assets/js/main.js @@ -6,3 +6,12 @@ document.querySelector(".navbar-burger").addEventListener("click", function() { // Fix sticky hover on iOS document.addEventListener("click", () => 0); + +// Address dynamic viewport units on mobile +function setCustomVh() { + let vh = window.innerHeight * 0.01; + document.documentElement.style.setProperty("--vh", `${vh}px`); +} + +window.addEventListener("resize", setCustomVh); +setCustomVh(); diff --git a/config/releases.exs b/config/releases.exs index 32bf38bb..f3b4d02f 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -19,6 +19,25 @@ defmodule Util do def parse_check_origin!("false"), do: false def parse_check_origin!(hosts) when is_binary(hosts), do: String.split(hosts, ",") def parse_check_origin!(hosts), do: raise("Invalid check_origin option: #{inspect(hosts)}") + + def validate_import_dir(nil), do: nil + + def validate_import_dir(path) do + case File.ls(path) do + {:ok, [_ | _]} -> + path + + {:ok, []} -> + nil + + {:error, :enoent} -> + nil + + {:error, reason} -> + IO.puts("Cannot access directory '#{path}': #{inspect(reason)}") + nil + end + end end config :teslamate, TeslaMate.Repo, @@ -27,7 +46,8 @@ config :teslamate, TeslaMate.Repo, database: System.fetch_env!("DATABASE_NAME"), hostname: System.fetch_env!("DATABASE_HOST"), port: System.get_env("DATABASE_PORT", "5432"), - pool_size: System.get_env("DATABASE_POOL_SIZE", "8") |> String.to_integer() + pool_size: System.get_env("DATABASE_POOL_SIZE", "10") |> String.to_integer(), + timeout: System.get_env("DATABASE_TIMEOUT", "60000") |> String.to_integer() config :teslamate, TeslaMateWeb.Endpoint, http: [:inet6, port: System.get_env("PORT", "4000")], @@ -55,3 +75,6 @@ config :logger, :console, metadata: [:car_id] config :teslamate, :srtm_cache, System.get_env("SRTM_CACHE", ".srtm_cache") + +config :teslamate, + import_directory: System.get_env("IMPORT_DIR", "import") |> Util.validate_import_dir() diff --git a/docs/configuration/environment_variables.md b/docs/configuration/environment_variables.md index c8c24866..b853b931 100644 --- a/docs/configuration/environment_variables.md +++ b/docs/configuration/environment_variables.md @@ -9,15 +9,17 @@ TeslaMate accepts the following environment variables for runtime configuration: | **DATABASE_NAME** | The database to connect to (**required**) | | | **DATABASE_HOST** | Hostname of the database server (**required**) | | | **DATABASE_PORT** | Port of the database server | 5432 | -| **DATABASE_POOL_SIZE** | Size of the database connection pool | 8 | +| **DATABASE_POOL_SIZE** | Size of the database connection pool | 10 | +| **DATABASE_TIMEOUT** | The time in milliseconds to wait for database query calls to finish | 60000 | | **VIRTUAL_HOST** | Host part used for generating URLs throughout the app | localhost | | **CHECK_ORIGIN** | Configures whether to check the origin header or not. May be `true` (**recommended**), `false` (_default_) or a comma-separated list of hosts that are allowed (e.g. `https://example.com,//another.com:8080`). Hosts also support wildcards. It defaults to true and, in such case, it will check against the host value in `VIRTUAL_HOST`. | false | | **PORT** | Port where the web interface is exposed | 4000 | | **DISABLE_MQTT** | Disables the MQTT feature if `true` | false | | **MQTT_HOST** | Hostname of the broker (**required** unless DISABLE_MQTT is `true`) | | -| **MQTT_USERNAME** | Username _(optional)_ | | -| **MQTT_PASSWORD** | Password _(optional)_ | | -| **MQTT_TLS** | Enables TLS if `true` _(optional)_ | false | -| **MQTT_TLS_ACCEPT_INVALID_CERTS** | Accepts invalid certificates if `true` _(optional)_ | false | -| **MQTT_NAMESPACE** | Inserts a custom namespace into the MQTT topic _(optional)_. For example, with `MQTT_NAMESPACE=account_0`: `teslamate/account_0/cars/$car_id/state`. | | +| **MQTT_USERNAME** | Username | | +| **MQTT_PASSWORD** | Password | | +| **MQTT_TLS** | Enables TLS if `true` | false | +| **MQTT_TLS_ACCEPT_INVALID_CERTS** | Accepts invalid certificates if `true` | false | +| **MQTT_NAMESPACE** | Inserts a custom namespace into the MQTT topic . For example, with `MQTT_NAMESPACE=account_0`: `teslamate/account_0/cars/$car_id/state`. | | +| **IMPORT_DIR** | The path of the directory for the import of data (e.g. TeslaFi) | ./import | | **TZ** | Used to establish the local time zone, e.g. to use the local time in logs. See [List of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). | | diff --git a/docs/import/teslafi.md b/docs/import/teslafi.md new file mode 100644 index 00000000..e3346f0b --- /dev/null +++ b/docs/import/teslafi.md @@ -0,0 +1,28 @@ +# Import from TeslaFi (BETA) + +## Requirements + +- **CREATE A [BACKUP](../maintenance/backup_restore.html) OF YOUR DATA!** + +- If you have been using TeslaMate prior to the release of version 1.16, the [docker-compose.yml](../installation/docker.html) needs to be updated. Add the following volume mapping to the `teslamate` service: + + ```YAML + services: + teslamate: + # ... + volumes: + - ./import:/opt/app/import + ``` + +- Export your TeslaFi data as CSV by month: `Settings -> Account -> "Download TeslaFi Data"` + +## Instructions + +1. Copy the exported CSV files into a **directory named `import`\*** next to the _docker-compose.yml_ +2. **Restart** the teslamate service and open the TeslaMate admin interface. Now the import form should be displayed instead of the vehicle summary. +3. Since the raw data is in the local timezone (assigned by the home address in the TeslaFi settings page) you need to **select your local timezone**. Then start the import. +4. On low-end hardware like the Raspberry Pi the import may take multiple hours, depending on the amount of data. If there is an overlap between the already existing TeslaMate and TeslaFi data, only the data prior to the first TeslaMate data will be imported. After the import is complete, remove or **empty the `import` directory** and **restart** the `teslamate` service. + +Since the exported CSV files do not contain addresses, they are added automatically afterwards. So please note that not all addresses are visible immediately after the import/restarting. Depending on the amount of data imported, it may take a while before they appear. The same applies to elevation data. + +_\* The path of the import directory can be customized via the [`IMPORT_DIR` environment variable](../configuration/environment_variables.html)._ diff --git a/docs/index.rst b/docs/index.rst index b0382b10..f8a33c73 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,12 @@ A powerful, self-hosted data logger for your Tesla. configuration/guides/tasker configuration/guides/macro_droid +.. toctree:: + :maxdepth: 1 + :caption: Import + + import/teslafi.md + .. toctree:: :maxdepth: 1 :caption: Integrations diff --git a/docs/installation/docker.md b/docs/installation/docker.md index 34b74723..ee028884 100644 --- a/docs/installation/docker.md +++ b/docs/installation/docker.md @@ -31,6 +31,8 @@ services: - MQTT_HOST=mosquitto ports: - 4000:4000 + volumes: + - ./import:/opt/app/import cap_drop: - all diff --git a/docs/installation/docker_advanced.md b/docs/installation/docker_advanced.md index 9aeb06c0..ab66fb7e 100644 --- a/docs/installation/docker_advanced.md +++ b/docs/installation/docker_advanced.md @@ -36,6 +36,8 @@ services: - VIRTUAL_HOST=${FQDN_TM} - CHECK_ORIGIN=true - TZ={$TM_TZ} + volumes: + - ./import:/opt/app/import labels: - 'traefik.enable=true' - 'traefik.port=4000' diff --git a/docs/installation/docker_advanced_apache.md b/docs/installation/docker_advanced_apache.md index 0f33d007..43756de8 100644 --- a/docs/installation/docker_advanced_apache.md +++ b/docs/installation/docker_advanced_apache.md @@ -37,6 +37,8 @@ services: - VIRTUAL_HOST=${FQDN_TM} - CHECK_ORIGIN=true - TZ={$TM_TZ} + volumes: + - ./import:/opt/app/import ports: - 127.0.0.1:4000:4000 cap_drop: diff --git a/lib/tesla_api/vehicle.ex b/lib/tesla_api/vehicle.ex index 48143790..63f2473a 100644 --- a/lib/tesla_api/vehicle.ex +++ b/lib/tesla_api/vehicle.ex @@ -23,18 +23,18 @@ defmodule TeslaApi.Vehicle do vehicle_state: nil def list(%Auth{token: token}) do - TeslaApi.get("/api/1/vehicles", token, transform: &vehicle/1) + TeslaApi.get("/api/1/vehicles", token, transform: &result/1) end def get(%Auth{token: token}, id) do - TeslaApi.get("/api/1/vehicles/#{id}", token, transform: &vehicle/1) + TeslaApi.get("/api/1/vehicles/#{id}", token, transform: &result/1) end def get_with_state(%Auth{token: token}, id) do - TeslaApi.get("/api/1/vehicles/#{id}/vehicle_data", token, transform: &vehicle/1) + TeslaApi.get("/api/1/vehicles/#{id}/vehicle_data", token, transform: &result/1) end - defp vehicle(v) do + def result(v) do %__MODULE__{ id: v["id"], vehicle_id: v["vehicle_id"], diff --git a/lib/teslamate/application.ex b/lib/teslamate/application.ex index bcdb7fe4..d700b60f 100644 --- a/lib/teslamate/application.ex +++ b/lib/teslamate/application.ex @@ -10,18 +10,37 @@ defmodule TeslaMate.Application do :ok = :telemetry.detach({Phoenix.Logger, [:phoenix, :socket_connected]}) :ok = :telemetry.detach({Phoenix.Logger, [:phoenix, :channel_joined]}) - [ - TeslaMate.Repo, - TeslaMate.Api, - TeslaMate.Locations, - TeslaMateWeb.Endpoint, - TeslaMate.Terrain, - TeslaMate.Vehicles, - if(mqtt_enabled?(), do: TeslaMate.Mqtt), - TeslaMate.Repair - ] - |> Enum.reject(&is_nil/1) - |> Supervisor.start_link(strategy: :one_for_one, name: TeslaMate.Supervisor) + Supervisor.start_link(children(), strategy: :one_for_one, name: TeslaMate.Supervisor) + end + + defp children do + mqtt_enabled? = !is_nil(Application.get_env(:teslamate, :mqtt)) + + case Application.get_env(:teslamate, :import_directory) do + nil -> + [ + TeslaMate.Repo, + TeslaMate.Api, + TeslaMate.Locations, + TeslaMateWeb.Endpoint, + TeslaMate.Terrain, + TeslaMate.Vehicles, + if(mqtt_enabled?, do: TeslaMate.Mqtt), + TeslaMate.Repair + ] + |> Enum.reject(&is_nil/1) + + import_directory -> + [ + TeslaMate.Repo, + TeslaMate.Api, + TeslaMate.Locations, + TeslaMateWeb.Endpoint, + {TeslaMate.Terrain, disabled: true}, + {TeslaMate.Repair, limit: 250}, + {TeslaMate.Import, directory: import_directory} + ] + end end # Tell Phoenix to update the endpoint configuration @@ -30,6 +49,4 @@ defmodule TeslaMate.Application do TeslaMateWeb.Endpoint.config_change(changed, removed) :ok end - - defp mqtt_enabled?, do: !is_nil(Application.get_env(:teslamate, :mqtt)) end diff --git a/lib/teslamate/import.ex b/lib/teslamate/import.ex new file mode 100644 index 00000000..6e5d1bcb --- /dev/null +++ b/lib/teslamate/import.ex @@ -0,0 +1,270 @@ +defmodule TeslaMate.Import do + use GenStateMachine + + require Logger + + alias TeslaMate.Settings.CarSettings + alias TeslaMate.Vehicles.Vehicle + alias TeslaMate.{Vehicles, Repair, Log} + alias TeslaMate.Log.{Car, State} + + alias __MODULE__.{Status, LineParser, FakeApi} + + defstruct( + path: nil, + files: [], + timezone: :utc, + error: nil, + completed: MapSet.new(), + car: nil, + pids: %{}, + deps: %{} + ) + + alias __MODULE__, as: Data + + defmodule Status do + defstruct(state: :idle, message: nil, files: []) + + def into(state, %Data{files: files, completed: completed}) do + files = + Enum.map(files, fn %{date: date} = file -> + complete = MapSet.member?(completed, date) + Map.put(file, :complete, complete) + end) + + case state do + {:error, reason} -> %__MODULE__{state: :error, message: reason, files: files} + state when is_atom(state) -> %__MODULE__{state: state, files: files} + end + end + end + + @name __MODULE__ + @topic "#{@name}/state" + + def start_link(opts) do + GenStateMachine.start_link(__MODULE__, opts, name: Keyword.get(opts, :name, @name)) + end + + def run(timezone), do: GenStateMachine.call(@name, {:run, timezone}) + def running?, do: GenStateMachine.call(@name, :running?) + def get_status, do: GenStateMachine.call(@name, :get_status) + def reload_directory, do: GenStateMachine.call(@name, :reload_directory) + def subscribe, do: Phoenix.PubSub.subscribe(TeslaMate.PubSub, @topic) + + @impl true + def init(opts) do + Process.flag(:trap_exit, true) + path = Keyword.fetch!(opts, :directory) + {:ok, :idle, %Data{path: path}, {:next_event, :internal, :read_directory}} + end + + ## Calls + + @impl true + def handle_event({:call, from}, {:run, tz}, :idle, data) do + {:next_state, :running, %Data{data | timezone: tz}, + [ + {:reply, from, :ok}, + {:next_event, :internal, :broadcast}, + {:next_event, :internal, :import} + ]} + end + + def handle_event({:call, from}, {:run, _tz}, _, _data) do + {:keep_state_and_data, {:reply, from, {:error, :not_allowed}}} + end + + def handle_event({:call, from}, :running?, state, _data) do + {:keep_state_and_data, {:reply, from, state == :running}} + end + + def handle_event({:call, from}, :get_status, state, data) do + {:keep_state_and_data, {:reply, from, Status.into(state, data)}} + end + + def handle_event({:call, from}, :reload_directory, _state, _data) do + {:keep_state_and_data, [{:reply, from, :ok}, {:next_event, :internal, :read_directory}]} + end + + ## Internal + + def handle_event(:internal, :broadcast, state, data) do + :ok = Phoenix.PubSub.broadcast(TeslaMate.PubSub, @topic, Status.into(state, data)) + :keep_state_and_data + end + + def handle_event(:internal, :read_directory, :idle, %Data{path: path} = data) do + case File.ls(path) do + {:error, reason} -> + {:next_state, {:error, reason}, {:next_event, :internal, :broadcast}} + + {:ok, files} -> + files = + files + |> Enum.map(fn n -> %{date: parse_fname(n), path: Path.join([path, n])} end) + |> Enum.reject(fn %{date: date} -> is_nil(date) end) + |> Enum.sort_by(fn %{date: date} -> date end) + + {:keep_state, %Data{data | files: files}, {:next_event, :internal, :broadcast}} + end + end + + def handle_event(:internal, :read_directory, _state, _data) do + :keep_state_and_data + end + + def handle_event(:internal, :import, :running, %Data{files: files} = data) do + Logger.info("Importing #{length(files)} file(s) ...") + + case create_evennt_streams(data) do + {:error, reason} -> + {:next_state, {:error, reason}, {:next_event, :internal, :broadcast}} + + {:ok, streams} -> + car = create_car(streams) + + :ok = Log.complete_current_state(car) + + date_limit = + with %State{start_date: date} <- Log.get_earliest_state(car) do + date + end + + api_name = :"api_#{car.name}" + + {:ok, api} = + FakeApi.start_link( + name: api_name, + event_streams: streams, + date_limit: date_limit, + pid: self() + ) + + {:ok, veh} = + Vehicle.start_link( + name: :"import_#{car.name}", + car: car, + import?: true, + deps_api: {FakeApi, api_name} + ) + + {:keep_state, %Data{data | car: car, pids: %{veh: veh, api: api}}, + {:next_event, :internal, :broadcast}} + end + end + + ## Info + + def handle_event(:info, {:done, chunk}, :running, %Data{completed: completed} = data) do + :ok = Repair.trigger_run() + + {:keep_state, %Data{data | completed: MapSet.put(completed, chunk)}, + {:next_event, :internal, :broadcast}} + end + + def handle_event(:info, :done, :running, %Data{car: car, pids: %{api: api, veh: veh}} = data) do + Logger.info("Import complete!") + + true = Process.exit(veh, :kill) + true = Process.exit(api, :normal) + + :ok = Log.complete_current_state(car) + :ok = Log.create_current_state(car) + :ok = Repair.trigger_run() + + {:next_state, :complete, data, {:next_event, :internal, :broadcast}} + end + + def handle_event(:info, {:EXIT, _from, :normal}, _state, _data), do: :keep_state_and_data + def handle_event(:info, {:EXIT, _from, :killed}, _state, _data), do: :keep_state_and_data + + def handle_event(:info, {:EXIT, _from, reason}, _state, data) do + Logger.warn("Import failed: #{inspect(reason, pretty: true)}") + {:next_state, {:error, reason}, data, {:next_event, :internal, :broadcast}} + end + + ## Private + + defp parse_fname(name) do + case name do + <<"TeslaFi"::bitstring, m::binary-size(2), y::binary-size(4), ".csv"::bitstring>> -> + parse_date(y, m) + + <<"TeslaFi"::bitstring, m::binary-size(1), y::binary-size(4), ".csv"::bitstring>> -> + parse_date(y, m) + + _ -> + nil + end + end + + defp parse_date(year, month) do + with {year, ""} <- Integer.parse(year), + {month, ""} <- Integer.parse(month) do + [year, month] + else + _ -> nil + end + end + + defp create_evennt_streams(%Data{files: files, timezone: tz}) do + alias TeslaApi.Vehicle.State.Drive + alias TeslaApi.Vehicle, as: Vec + + try do + event_streams = + files + |> Enum.sort_by(fn %{date: date} -> date end) + |> Enum.map(fn %{date: date, path: path} -> + stream = + path + |> File.stream!(read_ahead: 64 * 4096) + |> CSV.decode!(headers: true) + |> Task.async_stream(&LineParser.parse(&1, tz), timeout: :infinity, ordered: true) + |> Stream.filter(fn + {:ok, %Vec{state: "unknown"}} -> + false + + {:ok, %Vec{state: "online", drive_state: %Drive{latitude: lat, longitude: lng}}} -> + lat != nil and lng != nil + + {:ok, %Vec{drive_state: %Drive{timestamp: nil}}} -> + false + + {:ok, %Vec{}} -> + true + end) + + {date, stream} + end) + + {:ok, event_streams} + rescue + e in File.Error -> {:error, e.reason} + e -> {:error, e} + end + end + + defp create_car([]), do: raise("vehicle data is incomplete") + + defp create_car([{_date, %Stream{} = stream} | rest]) do + alias TeslaApi.Vehicle, as: Vec + + stream + |> Enum.find(fn {:ok, %Vec{id: eid, vehicle_id: vid, vin: vin}} -> + vin != nil and vid != nil and eid != nil + end) + |> case do + {:ok, vehicle} -> + car = Vehicles.create_or_update!(vehicle) + settings = %CarSettings{suspend_min: 0, suspend_after_idle_min: 99999} + + %Car{car | settings: settings} + + nil -> + create_car(rest) + end + end +end diff --git a/lib/teslamate/import/fake_api.ex b/lib/teslamate/import/fake_api.ex new file mode 100644 index 00000000..a7e891af --- /dev/null +++ b/lib/teslamate/import/fake_api.ex @@ -0,0 +1,91 @@ +defmodule TeslaMate.Import.FakeApi do + use GenServer + + require Logger + + alias TeslaApi.Vehicle.State.Drive + alias TeslaApi.Vehicle + + defstruct(pid: nil, events: [], event_streams: [], current_chunk: nil, date_limit: nil) + alias __MODULE__, as: State + + # API + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name), fullsweep_after: 10) + end + + def get_vehicle(name, _id) do + GenServer.call(name, :get_vehicle, :infinity) + end + + def get_vehicle_with_state(name, _id) do + GenServer.call(name, :get_vehicle_with_state, :infinity) + end + + # Callbacks + + @impl true + def init(opts) do + pid = Keyword.fetch!(opts, :pid) + event_streams = Keyword.fetch!(opts, :event_streams) + date_limit = Keyword.fetch!(opts, :date_limit) + + date_limit_ts = + with %DateTime{} <- date_limit do + DateTime.to_unix(date_limit) * 1000 + end + + {:ok, _ref} = :timer.send_interval(:timer.minutes(2), self(), :garbage_collect) + + {:ok, %State{pid: pid, event_streams: event_streams, date_limit: date_limit_ts}} + end + + @impl true + def handle_call(_action, _from, %State{date_limit: date_limit} = state) do + case pop(state) do + {:done, state} -> + send(state.pid, {:done, state.current_chunk}) + send(state.pid, :done) + {:noreply, state} + + {{:ok, %Vehicle{drive_state: %Drive{timestamp: ts}}}, state} + when ts >= date_limit -> + send(state.pid, {:done, state.current_chunk}) + send(state.pid, :done) + {:noreply, state} + + {{:ok, %Vehicle{}} = event, state} -> + {:reply, event, state} + end + end + + @impl true + def handle_info(:garbage_collect, state) do + :erlang.garbage_collect(self()) + {:noreply, state} + end + + ## Private + + def pop(%State{events: [], event_streams: []} = state) do + {:done, state} + end + + def pop(%State{events: [], event_streams: [{chunk, s} | streams]} = state) do + if state.current_chunk != nil, do: send(state.pid, {:done, state.current_chunk}) + + case Enum.into(s, []) do + [event | events] -> + {event, %State{state | events: events, event_streams: streams, current_chunk: chunk}} + + [] -> + Logger.warn("Processed empty chunk: #{inspect(chunk)}") + pop(%State{state | events: [], event_streams: streams, current_chunk: chunk}) + end + end + + def pop(%State{events: [event | events]} = state) do + {event, %State{state | events: events}} + end +end diff --git a/lib/teslamate/import/line_parser.ex b/lib/teslamate/import/line_parser.ex new file mode 100644 index 00000000..04758d87 --- /dev/null +++ b/lib/teslamate/import/line_parser.ex @@ -0,0 +1,125 @@ +defmodule TeslaMate.Import.LineParser do + use Timex + + require Logger + + alias TeslaApi.Vehicle.State.{Charge, Climate, Drive, VehicleConfig, VehicleState} + alias TeslaApi.Vehicle + + @default_vehicle %{ + "display_name" => "", + "charge_state" => %{}, + "climate_state" => %{}, + "drive_state" => %{}, + "vehicle_config" => %{}, + "vehicle_state" => %{} + } + + def parse(line, tz) when is_map(line) do + line + |> Enum.reduce(@default_vehicle, &into_vehicle(&1, &2, tz)) + |> Vehicle.result() + end + + @charge_state %Charge{} |> Map.keys() |> Enum.map(&to_string/1) + @climate_state %Climate{} |> Map.keys() |> Enum.map(&to_string/1) + @drive_state %Drive{} |> Map.keys() |> Enum.map(&to_string/1) + @vehicle %Vehicle{} |> Map.keys() |> Enum.map(&to_string/1) + @vehicle_config %VehicleConfig{} |> Map.keys() |> Enum.map(&to_string/1) + @vehicle_state %VehicleState{} |> Map.keys() |> Enum.map(&to_string/1) + + defp map_value(_, ""), do: nil + defp map_value(_, "None"), do: nil + defp map_value(_, "none"), do: nil + + defp map_value(_, "TRUE"), do: true + defp map_value(_, "True"), do: true + defp map_value(_, "true"), do: true + defp map_value(_, "FALSE"), do: false + defp map_value(_, "False"), do: false + defp map_value(_, "false"), do: false + + defp map_value("display_name", name), do: name + defp map_value("state", "waking"), do: "online" + defp map_value("state", "shutdown"), do: "online" + + @boolean ~w(battery_heater_on is_climate_on is_front_defroster_on is_rear_defroster_on + fast_charger_present not_enough_power_to_heat) + + defp map_value(key, val) when key in @boolean do + with v when v not in [nil, false, true] <- map_value(nil, val) do + nil + end + end + + defp map_value(_key, val) do + case Integer.parse(val) do + {i, ""} -> i + {_, _} -> to_float(val) + :error -> val + end + end + + defp to_float(val) do + case Float.parse(val) do + {f, ""} -> f + {_, _} -> val + :error -> val + end + end + + defp into_vehicle({key, val}, acc, tz) do + case {key, val} do + {"id", _val} -> + Map.put(acc, "id", :rand.uniform(65536)) + + {"Date", val} -> + {:ok, datetime} = + with {:error, _reason} <- Timex.parse(val, "{YYYY}-{M}-{D} {h24}:{m}:{s}"), + {:error, _reason} <- Timex.parse(val, "{M}/{D}/{YYYY} {h12}:{m}:{s} {AM}"), + {:error, _reason} <- Timex.parse(val, "{M}/{D}/{YYYY} {h24}:{m}") do + {:error, {:invalid_date_format, val}} + end + + ts = + case Timex.to_datetime(datetime, tz) do + %DateTime{} = dt -> + DateTime.to_unix(dt, :millisecond) + + {:error, reason} -> + Logger.warn( + "Could not convert date #{inspect(datetime)} w/ time zone #{inspect(tz)}: #{ + inspect(reason) + }" + ) + + nil + end + + ["vehicle_config", "vehicle_state", "drive_state", "climate_state", "charge_state"] + |> Enum.reduce(acc, fn key, acc -> put_in(acc, [key, "timestamp"], ts) end) + + {key, val} when key in @charge_state -> + put_in(acc, ["charge_state", key], map_value(key, val)) + + {key, val} when key in @climate_state -> + put_in(acc, ["climate_state", key], map_value(key, val)) + + {key, val} when key in @drive_state -> + put_in(acc, ["drive_state", key], map_value(key, val)) + + {key, val} when key in @vehicle_config -> + put_in(acc, ["vehicle_config", key], map_value(key, val)) + + {key, val} when key in @vehicle_state -> + put_in(acc, ["vehicle_state", key], map_value(key, val)) + + {key, val} when key in @vehicle -> + Map.put(acc, key, map_value(key, val)) + + {key, val} -> + Logger.debug("unhandled: #{inspect({key, val})}") + acc + end + end +end diff --git a/lib/teslamate/locations/geocoder.ex b/lib/teslamate/locations/geocoder.ex index e27b0cee..4b362675 100644 --- a/lib/teslamate/locations/geocoder.ex +++ b/lib/teslamate/locations/geocoder.ex @@ -42,7 +42,7 @@ defmodule TeslaMate.Locations.Geocoder do defp fetch(url, lang, params) do url = assemble_url(url, params) - case Mojito.get(url, headers(lang)) do + case Mojito.get(url, headers(lang), timeout: 15_000) do {:ok, %Response{status_code: 200, body: body}} -> {:ok, Jason.decode!(body)} {:ok, %Response{body: body}} -> {:error, Jason.decode!(body) |> Map.get("error")} {:error, %Error{reason: reason}} -> {:error, reason} diff --git a/lib/teslamate/log.ex b/lib/teslamate/log.ex index 8c00d2ae..b4efbf50 100644 --- a/lib/teslamate/log.ex +++ b/lib/teslamate/log.ex @@ -82,6 +82,58 @@ defmodule TeslaMate.Log do |> Repo.one() end + def create_current_state(%Car{id: id} = car) do + query = + from s in State, + where: s.car_id == ^id, + order_by: [desc: s.start_date], + limit: 1 + + with nil <- get_current_state(car), + %State{} = state <- Repo.one(query), + {:ok, _} <- state |> State.changeset(%{end_date: nil}) |> Repo.update() do + :ok + else + {:error, reason} -> {:error, reason} + _ -> :ok + end + end + + def complete_current_state(%Car{id: id} = car) do + case get_current_state(car) do + %State{start_date: date} = state -> + query = + from s in State, + where: s.car_id == ^id and s.start_date > ^date, + order_by: [asc: s.start_date], + limit: 1 + + end_date = + case Repo.one(query) do + %State{start_date: d} -> d + nil -> DateTime.add(date, 1, :second) + end + + with {:ok, _} <- + state + |> State.changeset(%{end_date: end_date}) + |> Repo.update() do + :ok + end + + nil -> + :ok + end + end + + def get_earliest_state(%Car{id: id}) do + State + |> where(car_id: ^id) + |> order_by(asc: :start_date) + |> limit(1) + |> Repo.one() + end + defp create_state(%Car{id: id}, attrs) do %State{car_id: id} |> State.changeset(attrs) diff --git a/lib/teslamate/release.ex b/lib/teslamate/release.ex index 2a1e7d9f..8a3ae3ec 100644 --- a/lib/teslamate/release.ex +++ b/lib/teslamate/release.ex @@ -1,6 +1,9 @@ defmodule TeslaMate.Release do @app :teslamate + import Ecto.Query + alias TeslaMate.Repo + def migrate do for repo <- repos() do {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true)) @@ -13,6 +16,15 @@ defmodule TeslaMate.Release do end end + def seconds_since_last_migration do + Repo.one( + from m in "schema_migrations", + select: fragment("EXTRACT(EPOCH FROM age(NOW(), ?::timestamp))::integer", m.inserted_at), + order_by: [desc: m.inserted_at], + limit: 1 + ) + end + defp repos do Application.load(@app) Application.fetch_env!(@app, :ecto_repos) diff --git a/lib/teslamate/repair.ex b/lib/teslamate/repair.ex index 72e4d138..5fc7baea 100644 --- a/lib/teslamate/repair.ex +++ b/lib/teslamate/repair.ex @@ -8,39 +8,63 @@ defmodule TeslaMate.Repair do alias TeslaMate.Locations.Address alias TeslaMate.{Repo, Locations} + defmodule State do + defstruct [:limit] + end + # API def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: __MODULE__) + GenServer.start_link(__MODULE__, opts, name: __MODULE__, fullsweep_after: 10) + end + + def trigger_run do + GenServer.cast(__MODULE__, :repair) end @impl true - def init(_opts) do - {:ok, _ref} = :timer.send_interval(:timer.hours(6), self(), :repair) + def init(opts) do + {:ok, _ref} = + opts + |> Keyword.get_lazy(:interval, fn -> :timer.hours(1) end) + |> :timer.send_interval(self(), :repair) - send(self(), :repair) + :ok = trigger_run() - {:ok, nil} + {:ok, %State{limit: Keyword.get(opts, :limit, 5000)}} end ## Repair @impl true - def handle_info(:repair, state) do + def handle_cast(:repair, %State{limit: limit} = state) do from(d in Drive, + join: sp in assoc(d, :start_position), + join: ep in assoc(d, :end_position), + select: [ + :id, + :car_id, + :start_date, + {:start_position, [:id, :latitude, :longitude]}, + {:end_position, [:id, :latitude, :longitude]} + ], where: (is_nil(d.start_address_id) or is_nil(d.end_address_id)) and (not is_nil(d.start_position_id) and not is_nil(d.end_position_id)), order_by: [desc: :id], - preload: [:start_position, :end_position] + preload: [start_position: sp, end_position: ep], + limit: ^limit ) |> Repo.all() |> repair() from(c in ChargingProcess, + join: p in assoc(c, :position), + select: [:id, :car_id, :start_date, {:position, [:id, :latitude, :longitude]}], where: is_nil(c.address_id) and not is_nil(c.position_id), order_by: [desc: :id], - preload: [:position] + preload: [position: p], + limit: ^limit ) |> Repo.all() |> repair() @@ -48,6 +72,7 @@ defmodule TeslaMate.Repair do {:noreply, state} end + @impl true def handle_info(msg, state) do Logger.warn("Unexpected message: #{inspect(msg, pretty: true)}") {:noreply, state} diff --git a/lib/teslamate/terrain.ex b/lib/teslamate/terrain.ex index 515eac51..a62b4938 100644 --- a/lib/teslamate/terrain.ex +++ b/lib/teslamate/terrain.ex @@ -27,18 +27,24 @@ defmodule TeslaMate.Terrain do def init(opts) do {:ok, client} = SRTM.Client.new(cache_path()) - timeout = Keyword.get(opts, :timeout, 100) - name = Keyword.get(opts, :name, @name) - - deps = %{ - srtm: Keyword.get(opts, :deps_srtm, SRTM), - log: Keyword.get(opts, :deps_log, Log) + data = %Data{ + timeout: Keyword.get(opts, :timeout, 100), + name: Keyword.get(opts, :name, @name), + client: client, + deps: %{ + srtm: Keyword.get(opts, :deps_srtm, SRTM), + log: Keyword.get(opts, :deps_log, Log) + } } - {:ok, _ref} = :timer.send_interval(:timer.hours(3), self(), :purge_srtm_in_memory_cache) + case Keyword.get(opts, :disabled, false) do + false -> + {:ok, _ref} = :timer.send_interval(:timer.hours(3), self(), :purge_srtm_in_memory_cache) + {:ok, :ready, data, {:next_event, :internal, {:fetch_positions, 0}}} - {:ok, :ready, %Data{client: client, timeout: timeout, deps: deps, name: name}, - {:next_event, :internal, {:fetch_positions, 0}}} + true -> + {:ok, :disabled, data} + end end ## Call @@ -78,7 +84,9 @@ defmodule TeslaMate.Terrain do {positions, next} -> Logger.info("Adding elevation to #{length(positions)} positions ...") + :ok = GenStateMachine.cast(self(), :process) + {:next_state, {:update, positions, next, nil}, data} end end @@ -86,7 +94,11 @@ defmodule TeslaMate.Terrain do ## Cast def handle_event(:cast, :process, {:update, [], next, nil}, data) do - {:next_state, :ready, data, {:next_event, :internal, {:fetch_positions, next}}} + {:next_state, :ready, data, + [ + {:next_event, :info, :purge_srtm_in_memory_cache}, + {:next_event, :internal, {:fetch_positions, next}} + ]} end def handle_event(:cast, :process, {:update, [%Position{} = p | rest], next, nil}, data) do @@ -149,7 +161,12 @@ defmodule TeslaMate.Terrain do def handle_event(:info, :purge_srtm_in_memory_cache, _state, %Data{client: client} = data) do Logger.debug("Purging SRTM in-memory cache ...") {:ok, client} = SRTM.Client.purge_in_memory_cache(client, keep: 2) - {:keep_state, %Data{data | client: client}} + {:keep_state, %Data{data | client: client}, {:next_event, :info, :garbage_collect}} + end + + def handle_event(:info, :garbage_collect, _state, _data) do + :erlang.garbage_collect(self()) + :keep_state_and_data end # Private diff --git a/lib/teslamate/vehicles.ex b/lib/teslamate/vehicles.ex index ed9da0ba..fb1ce5d1 100644 --- a/lib/teslamate/vehicles.ex +++ b/lib/teslamate/vehicles.ex @@ -94,7 +94,7 @@ defmodule TeslaMate.Vehicles do vehicles end - defp create_or_update!(%TeslaApi.Vehicle{} = vehicle) do + def create_or_update!(%TeslaApi.Vehicle{} = vehicle) do unless is_nil(name = vehicle.display_name), do: Logger.info("Starting logger for '#{name}'") {:ok, car} = diff --git a/lib/teslamate/vehicles/vehicle.ex b/lib/teslamate/vehicles/vehicle.ex index 80d6a2f4..61853125 100644 --- a/lib/teslamate/vehicles/vehicle.ex +++ b/lib/teslamate/vehicles/vehicle.ex @@ -38,12 +38,14 @@ defmodule TeslaMate.Vehicles.Vehicle do end model = - case String.downcase(type) do - "models" <> _ -> "S" - "model3" <> _ -> "3" - "modelx" <> _ -> "X" - "modely" <> _ -> "Y" - _____________ -> nil + with str when is_binary(str) <- type do + case String.downcase(str) do + "models" <> _ -> "S" + "model3" <> _ -> "3" + "modelx" <> _ -> "X" + "modely" <> _ -> "Y" + _____________ -> nil + end end {:ok, %{trim_badging: trim_badging, model: model, name: name}} diff --git a/lib/teslamate_web/controllers/car_controller.ex b/lib/teslamate_web/controllers/car_controller.ex index 9b4f0fbd..31036721 100644 --- a/lib/teslamate_web/controllers/car_controller.ex +++ b/lib/teslamate_web/controllers/car_controller.ex @@ -6,6 +6,7 @@ defmodule TeslaMateWeb.CarController do alias TeslaMate.Api, warn: false alias TeslaMate.{Log, Vehicles} + plug :redirect_if_importing when action in [:index] plug :fetch_signed_in when action in [:index] plug :redirect_unless_signed_in when action in [:index] @@ -44,8 +45,16 @@ defmodule TeslaMateWeb.CarController do _____ -> defp fetch_signed_in(conn, _opts), do: assign(conn, :signed_in?, Api.signed_in?()) end + defp redirect_if_importing(conn, _) do + case Application.get_env(:teslamate, :import_directory) do + nil -> conn + ___ -> conn |> redirect(to: import_page(conn)) |> halt() + end + end + defp redirect_unless_signed_in(%Plug.Conn{assigns: %{signed_in?: true}} = conn, _), do: conn defp redirect_unless_signed_in(conn, _opts), do: conn |> redirect(to: sign_in(conn)) |> halt() defp sign_in(conn), do: Routes.live_path(conn, TeslaMateWeb.SignInLive.Index) + defp import_page(conn), do: Routes.live_path(conn, TeslaMateWeb.ImportLive.Index) end diff --git a/lib/teslamate_web/controllers/donate_controller.ex b/lib/teslamate_web/controllers/donate_controller.ex new file mode 100644 index 00000000..1e3c8a5d --- /dev/null +++ b/lib/teslamate_web/controllers/donate_controller.ex @@ -0,0 +1,26 @@ +defmodule TeslaMateWeb.DonateController do + use TeslaMateWeb, :controller + + action_fallback TeslaMateWeb.FallbackController + + @url "https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=YE4CPXRAV9CVL&source=url" + @max_age 1 * 24 * 60 * 60 + + def index(conn, _opts) do + value = to_string(times_clicked(conn) + 1) + + conn + |> put_resp_cookie("donate", value, max_age: @max_age) + |> redirect(external: @url) + |> halt() + end + + defp times_clicked(conn) do + with %{cookies: %{"donate" => n}} <- conn, + {n, ""} <- Integer.parse(n) do + n + else + _ -> 0 + end + end +end diff --git a/lib/teslamate_web/live/import_live/index.ex b/lib/teslamate_web/live/import_live/index.ex new file mode 100644 index 00000000..a9acc2c1 --- /dev/null +++ b/lib/teslamate_web/live/import_live/index.ex @@ -0,0 +1,74 @@ +defmodule TeslaMateWeb.ImportLive.Index do + use Phoenix.LiveView + + defmodule Settings do + use Ecto.Schema + import Ecto.Changeset + + schema("settings", do: field(:timezone, :string)) + + def changeset(attrs), do: cast(%__MODULE__{}, attrs, [:timezone]) + def apply(changeset), do: apply_changes(changeset) + end + + alias TeslaMateWeb.ImportView + alias TeslaMate.Import + + @impl true + def render(assigns), do: ImportView.render("index.html", assigns) + + @impl true + def mount(_params, %{"settings" => _, "locale" => locale}, socket) do + if connected?(socket) do + Gettext.put_locale(locale) + :ok = Import.subscribe() + end + + socket = + socket + |> assign(status: Import.get_status()) + |> assign(changeset: Settings.changeset(%{timezone: get_time_zone()})) + |> assign(timezones: Timex.timezones()) + + {:ok, socket} + end + + @impl true + def handle_event("change", %{"settings" => attrs}, socket) do + {:noreply, assign(socket, changeset: Settings.changeset(attrs))} + end + + def handle_event("import", %{"settings" => attrs}, %{assigns: %{status: status}} = socket) do + %Settings{timezone: tz} = + attrs + |> Settings.changeset() + |> Settings.apply() + + :ok = Import.run(tz) + + {:noreply, assign(socket, status: %Import.Status{status | state: :running})} + end + + def handle_event("reload", _params, socket) do + :ok = Import.reload_directory() + {:noreply, socket} + end + + @impl true + def handle_info(%Import.Status{} = status, socket) do + {:noreply, assign(socket, status: status)} + end + + ## Private + + defp get_time_zone do + case Timex.local() do + %DateTime{time_zone: tz} -> tz + _ -> nil + end + rescue + _ -> + # https://github.com/bitwalker/timex/issues/521 + nil + end +end diff --git a/lib/teslamate_web/plugs/donate.ex b/lib/teslamate_web/plugs/donate.ex new file mode 100644 index 00000000..d2f84b9c --- /dev/null +++ b/lib/teslamate_web/plugs/donate.ex @@ -0,0 +1,35 @@ +defmodule TeslaMateWeb.Plugs.Donate do + import Plug.Conn + + alias TeslaMate.{Release, Import} + + @max_age 30 * 24 * 60 * 60 + + def init(opts), do: opts + + def call(%{req_cookies: %{"donate" => n}} = conn, _opts) when is_binary(n), do: conn + def call(conn, _opts), do: put_donate_cookie(conn) + + defp put_donate_cookie(conn) do + cond do + importing_data?() -> + put_resp_cookie(conn, "donate", "0", max_age: @max_age) + + Release.seconds_since_last_migration() < @max_age / 2 -> + put_resp_cookie(conn, "donate", "0", max_age: @max_age) + + true -> + conn + end + end + + defp importing_data? do + with pid when is_pid(pid) <- Process.whereis(Import), + true <- Process.alive?(pid), + %Import.Status{state: s} when s != :error <- Import.get_status() do + true + else + _ -> false + end + end +end diff --git a/lib/teslamate_web/router.ex b/lib/teslamate_web/router.ex index ee6b86c5..badb517d 100644 --- a/lib/teslamate_web/router.ex +++ b/lib/teslamate_web/router.ex @@ -1,7 +1,8 @@ defmodule TeslaMateWeb.Router do use TeslaMateWeb, :router - alias TeslaMate.Settings + alias TeslaMate.{Settings, Release, Import} + alias TeslaMateWeb.Plugs.Donate pipeline :browser do plug :accepts, ["html"] @@ -11,6 +12,7 @@ defmodule TeslaMateWeb.Router do plug Phoenix.LiveView.Flash plug :protect_from_forgery plug :put_secure_browser_headers + plug Donate plug :fetch_settings end @@ -28,6 +30,8 @@ defmodule TeslaMateWeb.Router do live "/geo-fences/new", GeoFenceLive.Form live "/geo-fences/:id/edit", GeoFenceLive.Form live "/charge-cost/:id", ChargeLive.Cost + live "/import", ImportLive.Index + get "/donate", DonateController, :index end scope "/api", TeslaMateWeb do diff --git a/lib/teslamate_web/templates/import/index.html.leex b/lib/teslamate_web/templates/import/index.html.leex new file mode 100644 index 00000000..64c26e35 --- /dev/null +++ b/lib/teslamate_web/templates/import/index.html.leex @@ -0,0 +1,80 @@ + + +
| <%= "#{month}/#{year}" %> | +<%= Path.absname(path) %> | ++ + <%= case {@status.state, complete} do %> + <% {:running, true} -> %> + <% {:running, false} -> %> <%= content_tag :span, nil, class: "is-loading" %> + <% {:error, true} -> %> + <% {:error, false} -> %> + <% {:complete, true } -> %> + <% {:complete, false} -> %> + <% {:idle, _} -> %> + <% end %> + + | +