Parse CSV files with nimble_csv

lib/teslamate/import.ex

@@ -8,7 +8,7 @@ defmodule TeslaMate.Import do
   alias TeslaMate.{Vehicles, Repair, Log}
   alias TeslaMate.Log.{Car, State}
 
-  alias __MODULE__.{Status, LineParser, FakeApi}
+  alias __MODULE__.{Status, LineParser, FakeApi, CSV}
 
   defstruct(
     path: nil,
@@ -218,10 +218,19 @@ defmodule TeslaMate.Import do
         files
         |> Enum.sort_by(fn %{date: date} -> date end)
         |> Enum.map(fn %{date: date, path: path} ->
-          stream =
           path
           |> File.stream!(read_ahead: 64 * 4096)
-            |> CSV.decode!(headers: true)
+          |> CSV.parse()
+          |> case do
+            {:error, :unsupported_delimiter} ->
+              raise "Unsupported delimiter"
+
+            {:error, :no_contents} ->
+              {date, Stream.map([], & &1)}
+
+            {:ok, rows} ->
+              stream =
+                rows
                 |> Task.async_stream(&LineParser.parse(&1, tz), timeout: :infinity, ordered: true)
                 |> Stream.filter(fn
                   {:ok, %Veh{state: "unknown"}} ->
@@ -238,6 +247,7 @@ defmodule TeslaMate.Import do
                 end)
 
               {date, stream}
+          end
         end)
 
       {:ok, event_streams}

lib/teslamate/import/csv.ex

@@ -0,0 +1,30 @@
+defmodule TeslaMate.Import.CSV do
+  NimbleCSV.define(Parser, separator: ",", escape: "\"", newlines: ["\r\n", "\n"])
+
+  def parse(file_stream) do
+    file_stream
+    |> Parser.parse_stream(skip_headers: false)
+    |> Enum.take(2)
+    |> case do
+      [[_], _] ->
+        {:error, :unsupported_delimiter}
+
+      [] ->
+        {:error, :no_contents}
+
+      [_] ->
+        {:error, :no_contents}
+
+      [headers, _] ->
+        rows =
+          file_stream
+          |> Parser.parse_stream()
+          |> Stream.flat_map(fn
+            [""] -> []
+            row -> [headers |> Enum.zip(row) |> Enum.into(%{})]
+          end)
+
+        {:ok, rows}
+    end
+  end
+end

mix.exs

@@ -56,7 +56,7 @@ defmodule TeslaMate.MixProject do
       {:mock, "~> 0.3", only: :test},
       {:castore, "~> 0.1"},
       {:ex_cldr, "~> 2.0"},
-      {:csv, "~> 2.3"},
+      {:nimble_csv, "~> 1.1"},
       {:timex, "~> 3.0"},
       {:websockex, "~> 0.4"},
       {:tzdata, "~> 1.0"},

mix.lock

@@ -6,12 +6,11 @@
   "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"},
   "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"},
   "cowlib": {:hex, :cowlib, "2.9.1", "61a6c7c50cf07fdd24b2f45b89500bb93b6686579b069a89f88cb211e1125c78", [:rebar3], [], "hexpm", "e4175dc240a70d996156160891e1c62238ede1729e45740bdd38064dad476170"},
-  "csv": {:hex, :csv, "2.4.1", "50e32749953b6bf9818dbfed81cf1190e38cdf24f95891303108087486c5925e", [:mix], [{:parallel_stream, "~> 1.0.4", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm", "54508938ac67e27966b10ef49606e3ad5995d665d7fc2688efb3eab1307c9079"},
   "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"},
   "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"},
-  "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"},
+  "ecto": {:hex, :ecto, "3.5.0", "9b45303af8e7eea81c0ad6fbcf2d442edb3f1c535a32ca42e3b1f31091a8995e", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9413a21b0b1a8256724545550832605918ab26632010bd47ce9d71d24f4f4bd1"},
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
-  "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"},
+  "ecto_sql": {:hex, :ecto_sql, "3.5.0", "760aa2935cc80b72da83fbd8cc97923623a2401915c308afea2cf2b0aabf4b2e", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3bab456e3ebb5680b327313f57ebb5356882a59fe04964a03232a83dc4c44aa2"},
   "ex_cldr": {:hex, :ex_cldr, "2.17.2", "b46d8e18ec159a7c3079d8d87a703bcf248878c0e190435f418ecb7738b6848a", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: true]}, {:cldr_utils, "~> 2.9", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:gettext, "~> 0.13", [hex: :gettext, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "b93a7ae5081221144696f058b9350812f72557de4c8664780d8cd07251e1ecc3"},
   "excoveralls": {:hex, :excoveralls, "0.13.2", "5ca05099750c086f144fcf75842c363fc15d7d9c6faa7ad323d010294ced685e", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1e7ed75c158808a5a8f019d3ad63a5efe482994f2f8336c0a8c77d2f0ab152ce"},
   "file_system": {:hex, :file_system, "0.2.9", "545b9c9d502e8bfa71a5315fac2a923bd060fd9acb797fe6595f54b0f975fd32", [:mix], [], "hexpm", "3cf87a377fe1d93043adeec4889feacf594957226b4f19d5897096d6f61345d8"},
@@ -30,9 +29,9 @@
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
   "mint": {:hex, :mint, "1.2.0", "65e9d75c60c456a5fb1b800febb88f061f56157d103d755b99fcaeaeb3e956f3", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "19cbb3a5be91b7df4a35377ba94b26199481a541add055cf5d1d4299b55125ab"},
   "mock": {:hex, :mock, "0.3.5", "feb81f52b8dcf0a0d65001d2fec459f6b6a8c22562d94a965862f6cc066b5431", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "6fae404799408300f863550392635d8f7e3da6b71abdd5c393faf41b131c8728"},
+  "nimble_csv": {:hex, :nimble_csv, "1.1.0", "b1dba4a86be9e03065c9de829050468e591f569100332db949e7ce71be0afc25", [:mix], [], "hexpm", "e986755bc302832cac429be6deda0fc9d82d3c82b47abefb68b3c17c9d949a3f"},
   "nimble_options": {:hex, :nimble_options, "0.3.0", "1872911bf50a048f04da26e02704e6aeafc362c2daa7636b6dbfda9492ccfcfa", [:mix], [], "hexpm", "180790a8644fea402452bc15bb54b9bf2c8e5c1fdeb6b39d8072e59c324edf7f"},
   "nimble_pool": {:hex, :nimble_pool, "0.2.1", "f1510446b63480794e6a71e42c092215e85638862d8ae994915f1a9cb56b4ae6", [:mix], [], "hexpm", "5a5b003eae0f9b62117ec6ac3e02f28a44daec4004ed2c7e2de720e797796c92"},
-  "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm", "639b2e8749e11b87b9eb42f2ad325d161c170b39b288ac8d04c4f31f8f0823eb"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
   "phoenix": {:hex, :phoenix, "1.5.5", "9a5a197edc1828c5f138a8ef10524dfecc43e36ab435c14578b1e9b4bd98858c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b10eaf86ad026eafad2ee3dd336f0fb1c95a3711789855d913244e270bde463b"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.2.1", "13f124cf0a3ce0f1948cf24654c7b9f2347169ff75c1123f44674afee6af3b03", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 2.15", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "478a1bae899cac0a6e02be1deec7e2944b7754c04e7d4107fc5a517f877743c0"},