From c536e413692e73f7a6ce83bd8b3b3e84a9063a44 Mon Sep 17 00:00:00 2001 From: anonymous Date: Wed, 29 Nov 2023 14:29:38 +0100 Subject: [PATCH] up --- .gitignore | 2 + aries/.gitignore | 11 ++ aries/.rspec | 3 + aries/Gemfile | 7 + aries/Gemfile.lock | 11 ++ aries/LICENSE.txt | 21 +++ aries/README.md | 39 +++++ aries/Rakefile | 8 + aries/aries.gemspec | 39 +++++ aries/bin/console | 15 ++ aries/bin/setup | 8 + aries/example.rb | 106 ++++++++++++ aries/lib/aries.rb | 113 +++++++++++++ aries/lib/aries/database.rb | 396 ++++++++++++++++++++++++++++++++++++++++++++ aries/lib/aries/version.rb | 5 + aries/spec/aries_spec.rb | 11 ++ aries/spec/spec_helper.rb | 15 ++ start.sh | 9 + 18 files changed, 819 insertions(+) create mode 100644 .gitignore create mode 100644 aries/.gitignore create mode 100644 aries/.rspec create mode 100644 aries/Gemfile create mode 100644 aries/Gemfile.lock create mode 100644 aries/LICENSE.txt create mode 100644 aries/README.md create mode 100644 aries/Rakefile create mode 100644 aries/aries.gemspec create mode 100755 aries/bin/console create mode 100755 aries/bin/setup create mode 100644 aries/example.rb create mode 100644 aries/lib/aries.rb create mode 100644 aries/lib/aries/database.rb create mode 100644 aries/lib/aries/version.rb create mode 100644 aries/spec/aries_spec.rb create mode 100644 aries/spec/spec_helper.rb create mode 100755 start.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eaee2a6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.pid +log.txt diff --git a/aries/.gitignore b/aries/.gitignore new file mode 100644 index 0000000..b04a8c8 --- /dev/null +++ b/aries/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/aries/.rspec b/aries/.rspec new file mode 100644 index 0000000..34c5164 --- /dev/null +++ b/aries/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/aries/Gemfile b/aries/Gemfile new file mode 100644 index 0000000..d41f271 --- /dev/null +++ b/aries/Gemfile @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } + +# gem "rails" diff --git a/aries/Gemfile.lock b/aries/Gemfile.lock new file mode 100644 index 0000000..17ca876 --- /dev/null +++ b/aries/Gemfile.lock @@ -0,0 +1,11 @@ +GEM + remote: https://rubygems.org/ + specs: + +PLATFORMS + ruby + +DEPENDENCIES + +BUNDLED WITH + 2.1.4 diff --git a/aries/LICENSE.txt b/aries/LICENSE.txt new file mode 100644 index 0000000..f9510bc --- /dev/null +++ b/aries/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 Luka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/aries/README.md b/aries/README.md new file mode 100644 index 0000000..8ef4565 --- /dev/null +++ b/aries/README.md @@ -0,0 +1,39 @@ +# Aries + +Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/aries`. To experiment with that code, run `bin/console` for an interactive prompt. + +TODO: Delete this and the text above, and describe your gem + +## Installation + +Add this line to your application's Gemfile: + +```ruby +gem 'aries' +``` + +And then execute: + + $ bundle install + +Or install it yourself as: + + $ gem install aries + +## Usage + +TODO: Write usage instructions here + +## Development + +After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. + +To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org). + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/aries. + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/aries/Rakefile b/aries/Rakefile new file mode 100644 index 0000000..b6ae734 --- /dev/null +++ b/aries/Rakefile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "rspec/core/rake_task" + +RSpec::Core::RakeTask.new(:spec) + +task default: :spec diff --git a/aries/aries.gemspec b/aries/aries.gemspec new file mode 100644 index 0000000..844d3ff --- /dev/null +++ b/aries/aries.gemspec @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require_relative "lib/aries/version" + +Gem::Specification.new do |spec| + spec.name = "aries" + spec.version = Aries::VERSION + spec.authors = ["Luka"] + spec.email = ["licina.luka@outlook.com"] + + spec.summary = "TODO: Write a short summary, because RubyGems requires one." + spec.description = "TODO: Write a longer description or delete this line." + spec.homepage = "TODO: Put your gem's website or public repo URL here." + spec.license = "MIT" + spec.required_ruby_version = ">= 2.6.0" + + spec.metadata["allowed_push_host"] = "TODO: Set to your gem server 'https://example.com'" + + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here." + spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here." + + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + spec.files = Dir.chdir(File.expand_path(__dir__)) do + `git ls-files -z`.split("\x0").reject do |f| + (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features)/|\.(?:git|travis|circleci)|appveyor)}) + end + end + spec.bindir = "exe" + spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } + spec.require_paths = ["lib"] + + # Uncomment to register a new dependency of your gem + # spec.add_dependency "example-gem", "~> 1.0" + + # For more information and examples about making a new gem, checkout our + # guide at: https://bundler.io/guides/creating_gem.html +end diff --git a/aries/bin/console b/aries/bin/console new file mode 100755 index 0000000..6652eee --- /dev/null +++ b/aries/bin/console @@ -0,0 +1,15 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require "aries" + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +# (If you use this, don't forget to add pry to your Gemfile!) +# require "pry" +# Pry.start + +require "irb" +IRB.start(__FILE__) diff --git a/aries/bin/setup b/aries/bin/setup new file mode 100755 index 0000000..dce67d8 --- /dev/null +++ b/aries/bin/setup @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +set -vx + +bundle install + +# Do any other automated setup that you need to do here diff --git a/aries/example.rb b/aries/example.rb new file mode 100644 index 0000000..246854d --- /dev/null +++ b/aries/example.rb @@ -0,0 +1,106 @@ +require "json" +require "csv" + +sentences = ['abcdefg', + 'abcdhfg', + 'abqwerty', + 'abqwerty'] + + +def maker charlist, index, data, trie + if index == (charlist.length) + return trie + end + + exists = trie.filter {|n| n.first == charlist[index] }.first + + node = (! exists) ? [ + charlist[index], # symbol, + [], # items + ] : exists + + node[1] = maker charlist, index + 1, data, node[1] + + if data && index == (charlist.length - 1) + if node[2] + node[2] << data + else + node[2] = [data] + end + end + + if ! exists + trie << node + end + + return trie +end + + +def fact value, data, trie + chars = value.to_s.chars + maker chars, 0, data, trie +end + + +initial_trie = [] + +sentences.each_with_index do |sentence, idx| + intial_trie = fact sentence, idx, initial_trie +end + +pp initial_trie + +initial_trie = [] + +File.readlines("/home/luka/Databases/aries/dissolver/db.csv").each_with_index do |row, idx| + if 1 > idx + next + end + + entity, attribute, value, timestmap, added = CSV.parse_line row + added = added == 'true' + + if 1 < value.to_s.length && value.to_s.length < 30 + initial_trie = fact "id_#{entity}", idx, initial_trie + initial_trie = fact "kv_#{attribute}_#{value}", entity, initial_trie + end +end + +initial_trie + +def find trie, phrase, index = 0 + if index == phrase.length - 1 + return trie.first.last + end + + node = trie.filter { |n| n.first == phrase[index] }.first + + if ! node + return nil + end + + return find(node[1], phrase, index + 1) +end + +criteria = { 'parent' => 'c282cb2a6395cc1d', + 'back' => nil } + + + +# x = find initial_trie, 'context_primary' # 'ede6dd2ccf3a0bfe' +results = {} + +criteria.each do |k, v| + + puts k + + if ! v.is_a?(NilClass) + sequence = "#{k}_#{v}" + results[k] = find initial_trie, sequence + end + +end + + +results diff --git a/aries/lib/aries.rb b/aries/lib/aries.rb new file mode 100644 index 0000000..f2468e5 --- /dev/null +++ b/aries/lib/aries.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true +require_relative "aries/version" +require_relative "aries/database" + +require "json" +require "socket" +# require "profile" + +module Aries + class Error < StandardError; end + # Your code goes here... + + def self.action callable + begin + [callable.call, nil] + rescue => e + pp e.backtrace << e + [nil, e] + end + end + + def self.valid? commands + for command in commands + command.keys.each do |k| + if ! ["add", "find", "first", "last"].include? k + return [nil, "Undefined command key '#{k}'"] + end + end + end + + return [true, nil] + end + + host = "localhost" + port = 1234 + server = TCPServer.new host, port + + puts "Server started, listening #{host}:#{port}" + + dbs = {} + db_factory = lambda do |n| + if ! dbs.keys.include? n + dbs[n] = Database.new n + puts "Initialized database #{n}" + end + + dbs[n] + end + + # ticker thread + Thread.new do + limit = 60 * 10 + loop do + dbs.keys.each do |n| + db = dbs[n] + if db.used_at and ((db.used_at.to_i + limit) < Time.now.to_i) + puts "Closing database #{n} it was last used more than #{limit} seconds ago" + dbs.delete n + puts "Closed database #{n}" + end + end + sleep 1 + end + end + + loop do + Thread.start server.accept do |client| + # client = server.accept + + connection = JSON.parse(client.gets) + db = db_factory.call connection['database'] + + # client.puts "Connected to Aries" + puts "Client connected #{client}" + + while commands = client.gets + commands, err = action -> { JSON.parse commands } + + if ! err.nil? + client.puts JSON.generate(err: err) + next + end + + _, err = valid? commands + + if ! err.nil? + client.puts JSON.generate(err: err) + next + end + + results = {} + commands.each_with_index do |command, idx| + + func, vals = command.to_a.first + begin + results[idx] = db.send(func, vals) + db.used_at = Time.now.to_i + rescue => e + pp [e.backtrace << e] + results = {err: e} + break + end + end + + client.puts JSON.generate(results) + end + + client.close + puts "Client disconnected #{client}" + end + end + +end diff --git a/aries/lib/aries/database.rb b/aries/lib/aries/database.rb new file mode 100644 index 0000000..d6a2f46 --- /dev/null +++ b/aries/lib/aries/database.rb @@ -0,0 +1,396 @@ +require "csv" +require "erb" +require "fileutils" +require "json" + +module Aries + + class Index + + attr_reader :path, :name + + def initialize path, name + @path = path + @name = name + end + + def refresh + puts "Refreshing index..." + File.readlines(File.join(@path, "#{@name}.csv")).each_with_index do |row, idx| + if 1 > idx + next + end + + entity, attribute, value, timestamp, added = CSV.parse_line row + self.commit idx, entity + self.commit idx, attribute + end + puts "Index refreshed!" + end + + def commit idx, target + fp = File.join @path, ".#{@name}.index.#{target}" + + if File.exists?(fp) and File.foreach(fp).any? { |ln| ln[idx.to_s] } + return + end + + File.write fp, "#{idx.to_s}\n", mode: 'a+' + end + + def check target + fp = File.join @path, ".#{@name}.index.#{target}" + + if ! File.exists?(fp) + return [] + end + + return File.readlines(fp).map &:to_i + end + end + + class Trie + attr_accessor :tree + + def initialize + @tree = [] + end + + def maker charlist, index, setter, trie + if index == (charlist.length) + return trie + end + + exists = trie.filter {|n| n.first == charlist[index] }.first + + node = (! exists) ? [ + charlist[index], # symbol, + [], # items + ] : exists + + node[1] = maker charlist, index + 1, setter, node[1] + + if setter && index == (charlist.length - 1) + node[2] = setter.call node[2] + end + + if ! exists + trie << node + end + + return trie + end + + def fact value, setter, trie + chars = value.to_s.chars + maker chars, 0, setter, trie + end + + def refresh path + puts "Refreshing trie..." + File.readlines(path).each_with_index do |row, idx| + if 1 > idx + next + end + + entity, attribute, value, timestmap, added = CSV.parse_line row + added = added == 'true' + + if 1 < value.to_s.length && value.to_s.length < 30 + setter = added ? lambda { |leaf| (leaf or []) + [entity] } : lambda { |leaf| (leaf or []) - [entity] } + + @tree = fact "id_#{entity}", lambda { |leaf| (leaf or []) << idx }, @tree + @tree = fact "kv_#{attribute}_#{value}", setter, @tree + end + end + puts "Trie refreshed!" + end + + def find trie, path, index = 0 + if index == path.length - 1 + return trie.first.last + end + + node = trie.filter { |n| n.first == path[index] }.first + + if ! node + return nil + end + + return find(node[1], path, index + 1) + end + + def ready? + return true + end + + def seek criteria, all = true + hits = [] + + criteria.each do |k, v| + if ! v.is_a?(NilClass) + sequence = "kv_#{k}_#{v}" + hit = find @tree, sequence + + if hit + hits += hit + end + end + end + + return hits + end + end + + class Database + + attr_reader :headers + attr_accessor :index + + attr_accessor :used_at + + # @version ? + def initialize path_part, name = nil + path = File.join(Dir.home, "Databases/aries", path_part) + FileUtils.mkdir_p path + + name = "db" unless name + + @headers = ['entity', 'attribute', 'value', 'timestamp', 'added'] + @path = File.join path, "#{name}.csv" + @index = Index.new path, name + + @trie = Trie.new + + if ! File.exists? @path + CSV.open @path, "w" do |fh| + fh << @headers + end + end + + @index.refresh + @trie.refresh @path + end + + # @brief appends to database + # @version ? + # + # @todo append to file instead of loading a CSV object? + def append entity, attribute, value, added = true + idx = (File.foreach(@path).inject(0) { |c, ln| c+1 }) + + CSV.open @path, "ab" do |fh| + @index.commit idx, entity + @index.commit idx, attribute + + # obviously this should be within some Trie method + if 1 < value.to_s.length && value.to_s.length < 30 + setter = added ? lambda { |leaf| (leaf or []) + [entity] } : lambda { |leaf| (leaf or []) - [entity] } + + @trie.tree = @trie.fact "id_#{entity}", lambda { |leaf| (leaf or []) << idx }, @trie.tree + @trie.tree = @trie.fact "kv_#{attribute}_#{value}", setter, @trie.tree + end + + time = Time.now.to_i + fh << [entity, attribute, value, time, added] + end + end + + # @version ? + def add eav + append eav['entity'], eav['attribute'], eav['value'] + end + + def readline position + line = nil + File.open(@path, 'r') do |f| + position.to_i.times { f.gets } + line = f.gets + end + + return line + end + + # @brief finds (latest) exact attribute + # @version ? + def check entity_id, key + rows = @index.check entity_id + attr = @index.check key + pos = rows & attr + val = nil # 'random-'.concat (('a'..'z').to_a + ('A'..'Z').to_a + (0..9).to_a).shuffle[0..4].join + + File.readlines(@path).each_with_index do |row, idx| + if 1 > idx + next + end + + if ! pos.include? idx + next + end + + entity, attribute, value, timestamp, added = CSV.parse_line row + added = JSON.parse added + val = added ? value : nil + end + + return val + end + + # @version ? + def get entity_id + created = updated = 0 + + rows = @index.check entity_id + + facts = {} + + File.readlines(@path).each_with_index do |row, idx| + if 1 > idx + next + end + + if ! rows.empty? + if ! rows.include? idx + next + end + end + + parts = CSV.parse_line row + entity, attribute, value, timestamp, added = parts + + if entity == entity_id + added = JSON.parse added + + facts[attribute] = added ? value : nil + created = [created, timestamp.to_i].min + updated = [updated, timestamp.to_i].max + end + + end + + facts['created'] = created + facts['updated'] = updated + facts['entity'] = entity_id + + return facts + end + + # @return Array + # + # @version ? + def scan criteria, all = true + hits = [] + seen = [] + + criteria = criteria.transform_keys &:to_sym + + File.readlines(@path).each_with_index do |row, idx| + if 1 > idx + next + end + + entity = row.split(',').first + if seen.include? entity + next + end + + seen.append entity + + entity, attribute, value, timestamp, added = CSV.parse_line row + added = added == 'true' + + matches = {} + + criteria.keys.each do |k| + v = criteria[k] + + if k == "$check" + matches[k] = ERB.new(criteria[k]).result(binding) # @todo - this does not work + end + + val = check entity, k + matches[k] = val == v + + # pp e: entity, q: criteria, curr: {k: k, v: v}, db: val, eq: val == v + end + + if matches.values.all? + record = get entity + record = record.transform_keys &:to_sym + + if ! all + return [record] + end + + hits.append record + end + end + + return hits + end + + def matching record, criteria + matches = {} + + criteria.keys.each do |k| + v = criteria[k] + + if k == "$check" + matches[k] = ERB.new(criteria[k]).result(binding) # @todo - this does not work + end + + val = record[k] + matches[k] = val == v + + # pp e: entity, q: criteria, curr: {k: k, v: v}, db: val, eq: val == v + end + + return matches.values.all? + end + + # @version ? + def query criteria, all = true + if criteria.empty? + return scan criteria, all + end + + if ! @trie.ready? + return scan criteria, all + end + + x = @trie.seek(criteria).uniq.map {|e| get(e)}.filter { |record| matching(record, criteria) }.map { |e| e.transform_keys(&:to_sym) } + + if ! all + return [x.first] + end + + return x + end + + # @version ? + def find criteria + query criteria + end + + # @version ? + def first criteria + matches = query criteria, false + if 1 > matches.length + return nil + end + + matches.first + end + + # @version ? + def last criteria + matches = query criteria, false + if 1 > matches.length + return nil + end + + matches.last + end + + end +end diff --git a/aries/lib/aries/version.rb b/aries/lib/aries/version.rb new file mode 100644 index 0000000..5849a69 --- /dev/null +++ b/aries/lib/aries/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module Aries + VERSION = "0.1.0" +end diff --git a/aries/spec/aries_spec.rb b/aries/spec/aries_spec.rb new file mode 100644 index 0000000..cfac135 --- /dev/null +++ b/aries/spec/aries_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +RSpec.describe Aries do + it "has a version number" do + expect(Aries::VERSION).not_to be nil + end + + it "does something useful" do + expect(false).to eq(true) + end +end diff --git a/aries/spec/spec_helper.rb b/aries/spec/spec_helper.rb new file mode 100644 index 0000000..3791b5e --- /dev/null +++ b/aries/spec/spec_helper.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require "aries" + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..260ab5e --- /dev/null +++ b/start.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# if [ -d "aries" ]; then +# cd aries && git pull && cd .. +# else +# git clone https://git.lain.church/apoc/aries +# fi + +ruby aries/lib/aries.rb