Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions apps/cf/config/config.exs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
# and its dependencies with the aid of the Config module.
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
use Mix.Config
import Config

# General application configuration
config :cf,
Expand Down Expand Up @@ -41,7 +41,7 @@ config :algoliax,
application_id: "N5GW2EAIFX"

# Import environment specific config
import_config "#{Mix.env()}.exs"
Config.import_config("#{Mix.env()}.exs")

config :cf,
openai_model: "gpt-4o"
Expand Down
4 changes: 2 additions & 2 deletions apps/cf/config/dev.exs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use Mix.Config
import Config

dev_secret = "8C6FsJwjV11d+1WPUIbkEH6gB/VavJrcXWoPLujgpclfxjkLkoNFSjVU9XfeNm6s"

Expand Down Expand Up @@ -32,5 +32,5 @@ config :cf, CF.Mailer, adapter: Bamboo.LocalAdapter

# Import local secrets if any - use wildcard to ignore errors
for config <- "*dev.secret.exs" |> Path.expand(__DIR__) |> Path.wildcard() do
import_config config
Config.import_config(config)
end
2 changes: 1 addition & 1 deletion apps/cf/lib/accounts/accounts.ex
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ defmodule CF.Accounts do
@request_validity 48 * 60 * 60

# Configure Fetching of user picture on Gravatar
@fetch_default_picture Application.get_env(:cf, :fetch_default_user_picture, true)
@fetch_default_picture Application.compile_env(:cf, :fetch_default_user_picture, true)

# ---- User creation ----

Expand Down
4 changes: 3 additions & 1 deletion apps/cf/lib/accounts/username_generator.ex
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
defmodule CF.Accounts.UsernameGenerator do
use Agent

@moduledoc """
Generates a unique username based on user id
"""

@name __MODULE__
@username_prefix "NewUser-"

def start_link do
def start_link(_opts \\ []) do
Agent.start_link(
fn ->
Hashids.new(
Expand Down
8 changes: 3 additions & 5 deletions apps/cf/lib/application.ex
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,14 @@ defmodule CF.Application do
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
import Supervisor.Spec

# Define workers and child supervisors to be supervised
children = [
# Other custom supervisors
supervisor(CF.Sources.Fetcher, []),
CF.Sources.Fetcher,
# Misc workers
worker(CF.Accounts.UsernameGenerator, []),
CF.Accounts.UsernameGenerator,
# Sweep tokens from db
worker(Guardian.DB.Token.SweeperServer, [])
Guardian.DB.Token.SweeperServer
]

# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
Expand Down
2 changes: 1 addition & 1 deletion apps/cf/lib/authenticator/oauth/facebook.ex
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ defmodule CF.Authenticator.OAuth.Facebook do
end

defp hmac(data, type, key) do
:crypto.hmac(type, key, data)
:crypto.mac(:hmac, type, key, data)
end

# ---- Private ----
Expand Down
23 changes: 1 addition & 22 deletions apps/cf/lib/errors/errors.ex
Original file line number Diff line number Diff line change
Expand Up @@ -36,30 +36,9 @@ defmodule CF.Errors do
end

@spec do_report(:error | :exit | :throw, any(), [any()], cf_error_params()) :: :ok
def do_report(type, value, stacktrace, params) do
def do_report(type, value, stacktrace, _params) do
# Any call to Sentry, Rollbar, etc. should be done here
Logger.error("[ERROR][#{type}] #{inspect(value)} - #{inspect(stacktrace)}")
:ok
end

defp build_occurence_data(params) do
default_occurrence_data()
|> add_user(params[:user])
|> Map.merge(params[:data] || %{})
end

defp default_occurrence_data() do
%{
"code_version" => CF.Application.version()
}
end

defp add_user(base, nil),
do: base

defp add_user(base, %{id: id, username: username}),
do: Map.merge(base, %{"person" => %{"id" => Integer.to_string(id), "username" => username}})

defp add_user(base, %{id: id}),
do: Map.merge(base, %{"person" => %{"id" => Integer.to_string(id)}})
end
5 changes: 1 addition & 4 deletions apps/cf/lib/llms/statements_creator.ex
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,6 @@ defmodule CF.LLMs.StatementsCreator do
end
end

@doc """
Chunk captions everytime we reach the max caption length
"""
defp chunk_captions(captions) do
# TODO: Add last captions from previous batch to preserve context
Enum.chunk_every(captions, @captions_chunk_size)
Expand Down Expand Up @@ -138,7 +135,7 @@ defmodule CF.LLMs.StatementsCreator do
defp create_statements_from_inputs(statements_inputs, video) do
inserted_at = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)

{nb_statements, statements} =
{_nb_statements, statements} =
DB.Repo.insert_all(
DB.Schema.Statement,
Enum.map(statements_inputs, fn %{"text" => text, "time" => time} ->
Expand Down
36 changes: 23 additions & 13 deletions apps/cf/lib/sources/fetcher.ex
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,25 @@ defmodule CF.Sources.Fetcher do

# ---- Public API ----

def start_link() do
import Supervisor.Spec
def child_spec(opts) do
%{
id: __MODULE__,
start: {__MODULE__, :start_link, [opts]},
type: :supervisor,
restart: :permanent,
shutdown: 2000
}
end

def start_link(_opts \\ []) do
Supervisor.start_link(
[
:hackney_pool.child_spec(
pool_name(),
timeout: @request_timeout,
max_connections: @max_connections
),
worker(CF.Sources.Fetcher.LinkChecker, [])
CF.Sources.Fetcher.LinkChecker
],
strategy: :one_for_all,
name: __MODULE__
Expand Down Expand Up @@ -52,15 +60,15 @@ defmodule CF.Sources.Fetcher do

def get_queue, do: Fetcher.LinkChecker.get_queue()

@url_regex ~r/^https?:\/\/[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&\/\/=]*)/

defp fetch(url, callback) do
without_domain = Regex.replace(@url_regex, url, "\\1")
path = Regex.replace(~r/\?.+$/, without_domain, "")
uri = URI.parse(url)

case do_fetch_source_metadata(url, MIME.from_path(path)) do
{:error, _} -> :error
{:ok, result} -> callback.(result)
case do_fetch_source_metadata(url, MIME.from_path(uri.path)) do
{:error, err} ->
:error

{:ok, result} ->
callback.(result)
end
end

Expand All @@ -69,13 +77,13 @@ defmodule CF.Sources.Fetcher do
defp do_fetch_source_metadata(url, mime_types) when mime_types in @fetchable_mime_types do
case HTTPoison.get(
url,
[],
[{"User-Agent", "CaptainFact/2.0"}],
follow_redirect: true,
max_redirect: 5,
hackney: [pool: pool_name()]
) do
{:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
{:ok, source_params_from_tree(Floki.parse(body))}
{:ok, source_params_from_tree(Floki.parse_document!(body))}

{:ok, %HTTPoison.Response{status_code: 404}} ->
{:error, :not_found}
Expand Down Expand Up @@ -137,10 +145,12 @@ defmodule CF.Sources.Fetcher do
# Link checker

defmodule LinkChecker do
use Agent

@doc """
Agent that record which links are currently fetched
"""
def start_link() do
def start_link(_opts \\ []) do
Agent.start_link(fn -> MapSet.new() end, name: Fetcher.link_checker_name())
end

Expand Down
1 change: 0 additions & 1 deletion apps/cf/lib/videos/captions_fetcher_youtube.ex
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
@behaviour CF.Videos.CaptionsFetcher

require Logger
import SweetXml

@user_agent "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/135.0"

Expand Down
2 changes: 1 addition & 1 deletion apps/cf/lib/videos/videos.ex
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ defmodule CF.Videos do
alias CF.Accounts.UserPermissions
alias CF.Videos.MetadataFetcher

@captions_fetcher Application.get_env(:cf, :captions_fetcher)
@captions_fetcher Application.compile_env(:cf, :captions_fetcher, nil)

@doc """
TODO with_speakers param is only required by REST API
Expand Down
6 changes: 3 additions & 3 deletions apps/cf_atom_feed/config/config.exs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
# and its dependencies with the aid of the Config module.
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
use Mix.Config
import Config

# General application configuration
config :cf_atom_feed,
Expand All @@ -20,4 +20,4 @@ config :db, DB.Repo, pool_size: 1

# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
Config.import_config("#{Mix.env()}.exs")
2 changes: 1 addition & 1 deletion apps/cf_atom_feed/config/dev.exs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use Mix.Config
import Config

# Do not include metadata nor timestamps in development logs
config :logger, :console, format: "[$level] $message\n"
Expand Down
12 changes: 6 additions & 6 deletions apps/cf_atom_feed/lib/application.ex
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ defmodule CF.AtomFeed.Application do
# See https://hexdocs.pm/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
import Supervisor.Spec

children = []
config = Application.get_env(:cf_atom_feed, CF.AtomFeed.Router)

if config[:cowboy] do
children = [supervisor(CF.AtomFeed.Router, []) | children]
end
children =
if config[:cowboy] do
[CF.AtomFeed.Router]
else
[]
end

# See https://hexdocs.pm/elixir/Supervisor.html
# for other strategies and supported options
Expand Down
12 changes: 11 additions & 1 deletion apps/cf_atom_feed/lib/router.ex
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,17 @@ defmodule CF.AtomFeed.Router do
plug(:match)
plug(:dispatch)

def start_link do
def child_spec(opts) do
%{
id: __MODULE__,
start: {__MODULE__, :start_link, [opts]},
type: :worker,
restart: :permanent,
shutdown: 500
}
end

def start_link(_opts \\ []) do
config = Application.get_env(:cf_atom_feed, CF.AtomFeed.Router)
Logger.info("Running CF.AtomFeed.Router with cowboy on port #{config[:cowboy][:port]}")
Plug.Cowboy.http(CF.AtomFeed.Router, [], config[:cowboy])
Expand Down
6 changes: 3 additions & 3 deletions apps/cf_graphql/config/config.exs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
# and its dependencies with the aid of the Config module.
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
use Mix.Config
import Config

# General application configuration
config :cf_graphql,
Expand All @@ -28,4 +28,4 @@ config :db, DB.Repo, pool_size: 5

# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
Config.import_config("#{Mix.env()}.exs")
2 changes: 1 addition & 1 deletion apps/cf_graphql/config/dev.exs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use Mix.Config
import Config

config :cf_graphql, CF.GraphQLWeb.Endpoint,
http: [port: 4002],
Expand Down
4 changes: 1 addition & 3 deletions apps/cf_graphql/lib/application.ex
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,12 @@ defmodule CF.Graphql.Application do
# See https://hexdocs.pm/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
import Supervisor.Spec

# Define workers and child supervisors to be supervised
children = [
# Start the PubSub system
{Phoenix.PubSub, name: CF.Graphql.PubSub},
# Start the endpoint when the application starts
supervisor(CF.GraphQLWeb.Endpoint, [])
{CF.GraphQLWeb.Endpoint, []}
]

# See https://hexdocs.pm/elixir/Supervisor.html
Expand Down
3 changes: 0 additions & 3 deletions apps/cf_graphql/lib/resolvers/statements.ex
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@ defmodule CF.Graphql.Resolvers.Statements do

alias Kaur.Result

import Ecto.Query
import Absinthe.Resolution.Helpers, only: [batch: 3]

alias DB.Repo
alias DB.Schema.Statement

Expand Down
3 changes: 0 additions & 3 deletions apps/cf_graphql/lib/resolvers/users.ex
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,6 @@ defmodule CF.Graphql.Resolvers.Users do
{:ok, user}
end

@doc """
Get logged in user
"""
def get_logged_in(_, _, _) do
{:ok, nil}
end
Expand Down
5 changes: 3 additions & 2 deletions apps/cf_graphql/lib/resolvers/videos.ex
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ defmodule CF.Graphql.Resolvers.Videos do
end
end

@deprecated "Use paginated_list/3"
# Deprecated: Use paginated_list/3 instead
# Keeping for backward compatibility with deprecated all_videos field
def list(_root, args, _info) do
Video
|> Video.query_list(Map.get(args, :filters, []), args[:limit])
Expand Down Expand Up @@ -126,7 +127,7 @@ defmodule CF.Graphql.Resolvers.Videos do
|> Ecto.Multi.update(:video, fn _repo ->
changeset
end)
|> Ecto.Multi.run(:action, fn _repo, %{video: video} ->
|> Ecto.Multi.run(:action, fn _repo, %{video: _video} ->
Repo.insert(CF.Actions.ActionCreator.action_update(user.id, changeset))
end)
|> Repo.transaction()
Expand Down
2 changes: 0 additions & 2 deletions apps/cf_graphql/lib/schema/schema.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ defmodule CF.Graphql.Schema do
use Absinthe.Schema
alias CF.Graphql.Resolvers
alias CF.Graphql.Schema.Middleware
import Absinthe.Resolution.Helpers, only: [dataloader: 1]

import_types(Absinthe.Plug.Types)

Expand Down Expand Up @@ -44,7 +43,6 @@ defmodule CF.Graphql.Schema do
query do
@desc "[Deprecated] Get all videos"
@deprecated "Please update to the paginated version (videos). This will be removed in 0.9."
@since "0.8.16"
field :all_videos, list_of(:video) do
arg(:filters, :video_filter)
arg(:limit, :integer)
Expand Down
Loading
Loading