def bob
10.times do
puts "B"
sleep 0.006
end
end
def alice
3.times do
puts "A"
sleep 0.01
end
raise "Alice aborted"
end
concurrent do |spindle|
spindle.spawn "Alice" do
alice
end
spindle.spawn "Bob", ->bob
end
def concurrent
spindle = Spindle.new
begin
yield spindle
ensure
spindle.spin
end
end
macro concurrent(first, *args)
concurrent do
spawn do
{{ first }}
end
{% for arg in args %}
spawn do
{{ arg }}
end
{% end %}
end
end
class Spindle
@fibers = [] of Fiber
@finished = Channel(Tuple(Fiber, Exception?)).new
def spawn(name = nil, &block : -> _)
spawn(name, block)
end
def spawn(name, proc)
puts "spindle spawning #{name}"
finished = @finished
fiber = Fiber.new(name) do
begin
proc.call
rescue exc
finished.send({Fiber.current, exc})
else
finished.send({Fiber.current, nil})
end
end
@fibers << fiber
Scheduler.enqueue fiber
end
def <<(fiber : Fiber)
@fibers << fiber
end
def spin
exception = nil
until @fibers.empty?
fiber, exc = @finished.receive
@fibers.delete fiber
case exc
when Fiber::CancelledException
puts "#{fiber.name} cancelled"
when Exception
@fibers.each &.cancel
puts "#{fiber.name} raised"
else
puts "#{fiber.name} finished"
end
exception = exc unless exception
end
raise exception if exception
end
end
class Fiber
# A `CancelRequestException` holds the callstack of `CancelledException` where
# `Fiber#cancel` was called.
class CancelRequestException < Exception
getter fiber : Fiber
def initialize(@callstack : CallStack, message : String = "Fiber cancel request")
super(message)
@fiber = Fiber.current
end
end
# A `CancelledException` is raised when a fiber is resumed after it was cancelled.
# See `Fiber#cancel` for details.
#
# If `cause` is `nil`, the fiber was cancelled while executing.
class CancelledException < Exception
getter fiber : Fiber
def initialize(@fiber : Fiber, cause : CancelRequestException? = nil)
super("Fiber cancelled: #{fiber}", cause)
end
def cause : CancelRequestException?
@cause.as(CancelRequestException?)
end
end
@cancel_request : CancelRequestException? = nil
# Stops this fiber from executing again and unwinds its stack.
#
# This method requests the fiber to raise a `CancelledException` the next time
# it is resumed and enqueues it in the scheduler. Therefore the unwinding will
# only take place the next time the scheduler reschedules.
#
# Raises `CancelledException` if this is the current fiber.
def cancel
if Fiber.current == self
# In case the current fiber is to be canceled, just raise the exception directly.
raise CancelledException.new self
else
# Otherwise register a cancel request, it will be evaluated on next resume.
@cancel_request ||= CancelRequestException.new(CallStack.new)
end
# Trigger scheduling
@resume_event.try &.free
Scheduler.enqueue(self)
end
def resume
previous_def
if cancel_request = Fiber.current.@cancel_request
raise CancelledException.new Fiber.current, cancel_request
end
end
end
This tries to solve the challenge of a fast parser and aggregator for a delimter-separated data file in [Crystal](https://crystal-lang.org).
* [Faster Command Line Tools in Go](https://aadrake.com/posts/2017-05-29-faster-command-line-tools-with-go.html)
* [Faster Command Line Tools in D](https://dlang.org/blog/2017/05/24/faster-command-line-tools-in-d/)
Here are some benchmarks on my fairly old laptop with Ubuntu on Linux (so total speed is quite low) and not comparable with the results from the linked benchmarks:
```
Naive 0.14 ( 7.37s ) (± 6.05%) 1.58× slower
Simple 0.12 ( 8.05s ) (± 5.71%) 1.73× slower
IntParser 0.2 ( 5.06s ) (± 4.63%) 1.09× slower
Array 0.21 ( 4.83s ) (± 6.63%) 1.04× slower
ArrayWithBackup 0.21 ( 4.66s ) (± 1.50%) fastest
Naive with cached stream 0.13 ( 7.62s ) (± 4.11%) 1.64× slower
Simple with cached stream 0.14 ( 7.08s ) (± 1.51%) 1.52× slower
IntParser with cached stream 0.19 ( 5.35s ) (±10.71%) 1.15× slower
Array with cached stream 0.2 ( 4.98s ) (± 7.27%) 1.07× slower
ArrayWithBackup with cached stream 0.2 ( 4.94s ) (± 7.10%) 1.06× slower
```
Running Crystal *IntParser* against *V4: Stop using strings* from the Go implementation yields the following results on my machine:
```
$ time ./crystal_int ngrams.tsv 1 2
max_key: 2006 sum: 22569013
real 0m6.689s
user 0m5.016s
sys 0m1.031s
$ time ./go_v4 ngrams.tsv 1 2
max_key: 2006 sum: 22569013
real 0m9.679s
user 0m8.016s
sys 0m1.438s
```
cli.cr
require "./faster"
file_path = ARGV[0]? || "./ngrams.tsv"
key_index = ARGV[1]?.try(&.to_i) || 1
value_index = ARGV[2]?.try(&.to_i) || 2
File.open(file_path) do |file|
result = {% if flag?("array_with_backup") %}Faster::ArrayWithBackup{% else %}Faster::IntParser{% end %}.process_data(file, key_index, value_index)
puts "max_key: #{result[0]} sum: #{result[1]}"
end
cli_bench.cr
require "./faster"
require "benchmark"
file_path = ARGV[0]? || "./ngrams.tsv"
key_index = ARGV[1]?.try(&.to_i) || 1
value_index = ARGV[2]?.try(&.to_i) || 2
IMPLEMENTATIONS = %w(Naive Simple IntParser Array ArrayWithBackup)
file = File.open(file_path)
Benchmark.ips(25, 0) do |x|
{% for implementation in IMPLEMENTATIONS %}
x.report({{ implementation }}) do
File.open(file_path) do |file|
Faster::{{implementation.id}}.process_data(file, key_index, value_index)
end
end
{% end %}
{% for implementation in IMPLEMENTATIONS %}
x.report("{{ implementation.id }} with cached stream") do
file.rewind
Faster::{{implementation.id}}.process_data(file, key_index, value_index)
end
{% end %}
end
at_exit do
file.close
end
cli_nuts.cr
require "./faster"
require "benchmark"
file_path = ARGV[0]? || "./ngrams.tsv"
key_index = ARGV[1]?.try(&.to_i) || 1
value_index = ARGV[2]?.try(&.to_i) || 2
data = IO::Memory.new
time_loading = Benchmark.measure do
File.open(file_path) do |file|
IO.copy(file, data)
data.rewind
end
end
puts "Time to load data into memory:"
puts time_loading
time_executing = Benchmark.measure do
result = {% if flag?("array_with_backup") %}Faster::ArrayWithBackup{% else %}Faster::IntParser{% end %}.process_data(data, key_index, value_index)
puts "max_key: #{result[0]} sum: #{result[1]}"
end
puts
puts "Time to process data:"
puts time_executing
faster.cr
module Faster
DELIMITER = '\t'
NL = '\n'
module Naive
def self.process_lines(io, key_index, value_index)
io.each_line do |line|
fields = line.split(DELIMITER)
yield fields[key_index], fields[value_index]
end
end
def self.process_data(io, key_index, value_index)
sums = {} of Int32 => Int32
process_lines(io, key_index, value_index) do |key, value|
key = key.to_i
sums[key] = sums.fetch(key, 0) + value.to_i
end
sums.max_by &.[](1)
end
end
module Simple
def self.process_lines(io, key_index, value_index)
field_index = 0
key = IO::Memory.new
value = IO::Memory.new
io.each_char do |c|
case c
when NL
yield key.to_s, value.to_s
key.clear
value.clear
field_index = 0
when DELIMITER
field_index += 1
else
case field_index
when key_index
key << c
when value_index
value << c
end
end
end
end
def self.process_data(io, key_index, value_index)
sums = {} of Int32 => Int32
process_lines(io, key_index, value_index) do |key, value|
key = key.to_i
sums[key] = sums.fetch(key, 0) + value.to_i
end
sums.max_by &.[](1)
end
end
module IntParser
def self.process_lines(io, key_index, value_index)
field_index = 0
key = 0
value = 0
io.each_char do |c|
case c
when '\n'
yield key, value
key = 0
value = 0
field_index = 0
when DELIMITER
field_index += 1
else
case field_index
when key_index
key = key * 10 + c.to_i
when value_index
value = value * 10 + c.to_i
end
end
end
end
def self.process_data(io, key_index, value_index)
sums = {} of Int32 => Int32
process_lines(io, key_index, value_index) do |key, value|
sums[key] = sums.fetch(key, 0) + value
end
sums.max_by &.[](1)
end
end
module Array
def self.process_lines(io, key_index, value_index)
field_index = 0
key = 0
value = 0
io.each_char do |c|
case c
when '\n'
yield key, value
key = 0
value = 0
field_index = 0
when DELIMITER
field_index += 1
else
case field_index
when key_index
key = key * 10 + c.to_i
when value_index
value = value * 10 + c.to_i
end
end
end
end
def self.process_data(io, key_index, value_index)
sums = ::Array(Int32).new(4096, 0)
process_lines(io, key_index, value_index) do |key, value|
sums[key] += value
end
max = {-1, 0}
sums.each_with_index do |value, key|
max = {key, value} if value > max[1]
end
max
end
end
module ArrayWithBackup
ARRAY_SIZE = 4096
def self.process_lines(io, key_index, value_index)
field_index = 0
key = 0
value = 0
io.each_char do |c|
case c
when '\n'
yield key, value
key = 0
value = 0
field_index = 0
when DELIMITER
field_index += 1
else
case field_index
when key_index
key = key * 10 + c.to_i
when value_index
value = value * 10 + c.to_i
end
end
end
end
def self.process_data(io, key_index, value_index)
sums = ::Array(Int32).new(ARRAY_SIZE, 0)
sums_hash = {} of Int32 => Int32
process_lines(io, key_index, value_index) do |key, value|
if(key >= ARRAY_SIZE)
sums_hash[key] = sums_hash.fetch(key, 0) + value
else
sums[key] += value
end
end
max = {-1, 0}
max = sums_hash.max_by &.[](1) unless sums_hash.empty?
sums.each_with_index do |value, key|
max = {key, value} if value > max[1]
end
max
end
end
end
macro memoize(type_decl, &block)
@{{type_decl.var}} : {{ type_decl.type }} | UninitializedMemo = UninitializedMemo::INSTANCE
def {{type_decl.var}}
if (value = @{{type_decl.var}}).is_a?(UninitializedMemo)
@{{type_decl.var}} = begin
{{block.body}}
end
else
value
end
end
end
class UninitializedMemo
INSTANCE = new
end
class ClassName
memoize expensive_query : Int32 do
puts "hello"
"hello".size
end
end
puts ClassName.new.expensive_query
module TuentiChallenge1
class Exercise
property file : String
def initialize(input_file)
@file = input_file
end
def run
lines = File.read_lines(file)
lines.each_with_index do |line, index|
next if index == 0
puts "Case ##{index}: #{tables_necessary(line.to_i)}"
end
end
def tables_necessary(diners)
if diners <= 4 && diners > 0
return 1
elsif diners <= 0
return 0
end
(((diners / 2.to_f) - 1).ceil).to_i
end
end
end
tc = TuentiChallenge1::Exercise.new("submitInput")
tc.run
ifdef windows
# already required "./winapi/kernel32.cr" in prelude
else
require "./dl/lib_dl.cr"
end
# This is a frontend wrapper,
# using LibDL or WinApi as backend.
module DL
ifdef windows
@[AlwaysInline]
def self.open(path)
WinApi.load_library(path)
end
# If specifics flags, calling LoadLibraryEx instead.
@[AlwaysInline]
def self.open(path, flags = 0)
WinApi.load_library_ex(path, nil, flags)
end
@[AlwaysInline]
def self.sym(handle, name)
WinApi.get_proc_address(handle, name)
end
else
@[AlwaysInline]
def self.open(path, mode = LibDL::LAZY | LibDL::GLOBAL)
LibDL.dlopen(path, mode)
end
@[AlwaysInline]
def self.sym(handle, symbol)
LibDL.dlsym(handle, symbol)
end
@[AlwaysInline]
def self.addr(addr, info)
LibDL.dladdr(addr, info)
end
end
end
module OS
module Windows
lib LibC
fun printf(format : UInt32*, args : UInt32) : NoReturn
# and other LibC bindings
end
module CWrapper
extend self
macro def printf(format, args) : Nil
LibC.printf(format.to_uint32, args)
nil
end
end
end
end
stdlib.cr
require "./c_wrapper"
module Kernel
def printf(format, args)
# Call platform indepedent method
CWrapper.printf(format, args)
end
end
# Call platform-specific method
ifdef linux
OS::Linux::CWrapper.linux_only_method _args
end
linux_c_wrapper.cr
module OS
module Linux
lib LibC
fun printf(format : UInt8*, args : UInt32) : NoReturn
# and other LibC bindings
end
module CWrapper
extend self
macro def printf(format, args) : Nil
LibC.printf(format, args)
nil
end
macro def linux_only_method(args) : Int32
# some lib call
end
end
end
end
lib_c.cr
lib LibC
# Common bindings go here
fun chdir(...)
fun printf(...)
fun wprintf(...)
fun atoi(...)
fun watoic(...)
end
ifdef windows
require "./windows_c_wrapper"
elsdef linux
require "./linux_c_wrapper"
end
c_wrapper.cr
ifdef linux
require "./linux_c_wrapper"
elsif windows
require "./windows_c_wrapper"
end
module CWrapper
ifdef linux
IMPL = OS::Linux::CWrapper
elsif windows
IMPL = OS::Linux::CWrapper
end
extend self
# This is just an example.
macro def printf(format, args) : Nil
{{IMPL}}.printf(format, args)
end
end
module ClassProperty
macro class_property(*names)
class_getter {{*names}}
class_setter {{*names}}
end
macro class_property?(*names)
class_getter? {{*names}}
class_setter {{*names}}
end
macro class_property!(*names)
getter! {{*names}}
setter {{*names}}
end
macro class_setter(*names)
{% for name in names %}
def self.{{name}}=(value)
@@{{name}} = value
end
{% end %}
end
macro class_getter(*names)
{% for name in names %}
def self.{{name}}
@@{{name}}
end
{% end %}
end
macro class_getter?(*names)
{% for name in names %}
def {{name}}?
@@{{name}}
end
{% end %}
end
macro class_getter!(*names)
{% for name in names %}
def {{name}}?
@@{{name}}
end
def {{name}}
@@{{name}}.not_nil!
end
{% end %}
end
end