try fixing extremely weird bug

apps/amethyst/lib/apps/connection_handler.ex

@@ -85,12 +85,16 @@ defmodule Amethyst.ConnectionHandler do
             end
           chunks = MapSet.new(visible_chunks_from(elem(cp, 0), elem(cp, 1), Map.get(state, :view_distance, 16)))
           new_chunks = MapSet.difference(chunks, prev_chunks)
-          # Process these chunks, we can process all chunks in parallel
-          Task.Supervisor.async_stream(state |> Map.get(:game) |> Map.get(:refs) |> Map.get(:task_supervisor),
-            new_chunks,
-            fn chunk -> process_chunk(chunk, state) end,
-            [ordered: false]
-          ) |> Stream.run()
+          # We can process all chunks in parallel
+          # me = self()
+          # Task.Supervisor.async_stream(state |> Map.get(:game) |> Map.get(:refs) |> Map.get(:task_supervisor),
+          #   new_chunks,
+          #   fn chunk -> process_chunk(me, chunk, state) end,
+          #   [ordered: false]
+          # ) |> Stream.run()
+          if new_chunks |> Enum.map(&process_chunk(self(), &1, state)) |> Enum.any?(&(&1 != :ok)) do
+            Logger.error("something happened while processing chunks")
+          end
         end
         loop(socket, connstate, version, state)
       {:send_packet, packet} ->
@@ -119,28 +123,91 @@ defmodule Amethyst.ConnectionHandler do
     end)
   end
 
-  defp process_chunk(chunk, state) do
+  defp process_chunk(to, chunk, state) do
     import Amethyst.NBT.Write
     alias Amethyst.Minecraft.Write
 
     chunk_array = Amethyst.API.Game.chunk(Map.get(state, :game), chunk)
+    {cx, cz} = chunk
 
     # TODO: Actually do heightmaps
+    # TODO: Doing all this processing could be at home somewhere else
     heightmaps = compound(%{})
 
     data = Enum.chunk_every(chunk_array, 16, 16, 0) # 0 -> air
-        |> Enum.reduce(fn chunk_section, acc ->
+        |> Enum.reduce("", fn chunk_section, acc ->
           blocks = chunk_section |> List.flatten()
-          block_count = blocks
-            |> Enum.map(&(if &1 == 0, do: 0, else: 1))
-            |> Enum.sum()
+          block_count = blocks |> Enum.filter(&(&1 != 0)) |> length
 
-          # TODO: Proper paletting
-          acc <> Write.short(block_count) <> Write.ubyte(15)
+          # Put together the palette
+          unique_blocks = MapSet.new(blocks)
+          min_bpe = MapSet.size(unique_blocks) |> :math.log2() |> ceil()
+          paletted_container_data = case min_bpe do
+            0 -> <<0::8>> <>
+            # SINGLE VALUED
+              Write.varint(MapSet.to_list(unique_blocks) |> List.first()) <>
+              Write.varint(0) # No data, empty pallette
+            min_bpe when min_bpe in 1..8 ->
+            # INDIRECT
+              # Minimum bpe accepted by minecraft is 4
+              bpe = max(min_bpe, 4)
+              palette = MapSet.to_list(unique_blocks) |>
+                Enum.with_index() |>
+                Map.new(fn {i, v} -> {v, i} end)
+              paletted_blocks = blocks |>
+                Enum.map(&(Map.get(palette, &1)))
+              Logger.debug("got here")
+              paletted_data = long_aligned_bit_string_reduce(paletted_blocks, bpe)
+
+              Write.ubyte(bpe) <>
+              Write.varint(map_size(palette)) <>
+              Enum.reduce(0..map_size(palette), "", fn i, acc ->
+                acc <> Write.varint(Map.get(palette, i))
+              end) <>
+              Write.varint(floor(bit_size(paletted_data) / 64)) <>
+              paletted_data
+            _ ->
+            # DIRECT
+              Logger.debug("got here with bpe #{min_bpe}")
+              data = long_aligned_bit_string_reduce(blocks, 15)
+              Write.ubyte(15) <>
+              Write.varint(floor(bit_size(data) / 64)) <>
+              data
+          end
+
+          # TODO: Send biome data, if that even makes sense
+          acc <> Write.short(block_count) <> paletted_container_data <> <<0::8, 0::8>>
         end)
 
-    # TODO: Use paletting
+      Logger.debug("Asking to send chunk packet...")
+      send(to, {:send_packet, %{
+        packet_type: :chunk_data_and_update_light,
+        chunk_x: cx, chunk_z: cz,
+        heightmaps: heightmaps,
+        data: data,
+        block_entities: [],
+        # TODO: Light
+        sky_light_mask: <<0>>,
+        block_light_mask: <<0>>,
+        empty_sky_light_mask: <<0>>,
+        empty_block_light_mask: <<0>>,
+        sky_light_arrays: [],
+        block_light_arrays: []
+      }})
+      :ok
+  end
 
+  defp long_aligned_bit_string_reduce(values, bpe) do
+    values |> Enum.reduce("", fn value, acc ->
+      next = acc <> <<value::size(bpe)-big>>
+      # man i hope they dont suddenly change the size of a long
+      if rem(bit_size(next), 64) + bpe < 64 do
+        # gotta pad it
+        next <> <<0::big-size(64 - rem(bit_size(next), 64))>>
+      else
+        next
+      end
+    end)
   end
 
   defp handle_packet(id, data, connstate, version, state) do

mix.lock

@@ -2,6 +2,7 @@
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"},
+  "elixir_math": {:hex, :elixir_math, "0.1.2", "5655bdf7f34e30906f31cdcf3031b43dd522ce8d2936b60ad4696b2c752bf5c9", [:mix], [], "hexpm", "34f4e4384903097a8ec566784fa8e9aa2b741247d225741f07cc48250c2aa64c"},
   "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"},
   "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},