File Coverage

blib/lib/Metabase/Archive/SQLite/Sharded.pm

Criterion	Covered	Total	%
statement	53	55	96.3
branch	6	10	60.0
condition	4	12	33.3
subroutine	15	15	100.0
pod	0	5	0.0
total	78	97	80.4

use 5.006;
use strict;
use warnings;

package Metabase::Archive::SQLite::Sharded;
# ABSTRACT: Metabase storage using multiple SQLite databases

our $VERSION = '1.001';

use Moose;
use Data::Stream::Bulk::Callback;
use Moose::Util::TypeConstraints;
use Metabase::Archive::SQLite;

with 'Metabase::Backend::SQLite';
with 'Metabase::Archive';

subtype 'ShardSize',
    as 'Int',
    where { $_ > 0 && $_ < 8 },
    message { "The number you provided, $_, was not between 1 and 8" };

has shard_digits => (
  is => 'ro',
  isa => 'ShardSize',
  default => 2,
);

has _shards => (
  is => 'ro',
  traits => ['Hash'],
  isa => 'HashRef[Object]',
  default => sub { return {} },
  handles => {
    '_get_shard' => 'get',
    '_set_shard' => 'set',
    '_all_shards' => 'values',
  },
);

sub initialize { }

sub _create_shard {
  my ($self, $key) = @_;

  my $filename = $self->filename;
  my ($basename, $ext) = $self->filename =~ m{^(.*)\.([^.]+)$};
  if ( $basename && $ext ) {
    $filename = "$basename\_$key.$ext";
  }
  else {
    $filename .= "_$key";
  }
  my %args = (
    filename => $filename,
    ($self->has_page_size ? ( page_size => $self->page_size ) : ()),
    ($self->has_cache_size ? ( cache_size => $self->cache_size ) : ()),
  );
  my $archive = Metabase::Archive::SQLite->new( %args );
  $archive->initialize;
  $self->_set_shard($key, $archive);
  return $archive;
}

sub _shard_key {
  my ($self, $guid) = @_;
  return substr $guid, (7-$self->shard_digits), $self->shard_digits;
}

sub store {
  my ( $self, $fact_struct ) = @_;
  my $key = $self->_shard_key( $fact_struct->{metadata}{core}{guid} );
  my $archive = $self->_get_shard($key) || $self->_create_shard($key);
  return $archive->store($fact_struct);
}


sub extract {
  my ( $self, $guid ) = @_;
  my $key = $self->_shard_key( $guid );
  my $archive = $self->_get_shard($key) || $self->_create_shard($key);
  return $archive->extract($guid);
}

sub delete {
  my ( $self, $guid ) = @_;
  my $key = $self->_shard_key( $guid );
  my $archive = $self->_get_shard($key) || $self->_create_shard($key);
  return $archive->delete($guid);
}

sub iterator {
  my ($self) = @_;

  my @iters = map { $_->iterator } $self->_all_shards;

  return Data::Stream::Bulk::Callback->new(
    callback => sub {
      if ( @iters ) {
        my $s = shift @iters;
        if ( my $next = $s->next ) {
          push @iters, $s; # round-robin
          return $next;
        }
      } else {
        return; # done
      }
    },
  );
}

1;

__END__

=pod

=encoding UTF-8

=head1 NAME

Metabase::Archive::SQLite::Sharded - Metabase storage using multiple SQLite databases

=head1 VERSION

version 1.001

=head1 SYNOPSIS

  use Metabase::Archive::SQLite::Sharded;

  my $archive = Metabase::Archive::SQLite::Sharded->new(
    filename => $sqlite_file,
    shard_digits => 2,
  );

=head1 DESCRIPTION

This is an implementation of the L<Metabase::Archive::SQL> role using SQLite
shards.

SQLite stores a database entirely in a single file.  That starts to become
slow as the size of the file gets large.  This Metabase::Archive shards
facts across multiple SQLite files.

It takes the same options as L<Metabase::Archive::SQLite>, with one additional
option, C<shard_digits>.  The C<shard_digits> attribute defines how many digits
of the GUID to use as a shard key.  Each digit is a hexadecimal number, so
digits increase the number of shards as a power of 16.  E.g., "1" means 16
shards, "2" means 256 shards and so on.

The shard key is inserted to the database C<filename> parameter either before
the final period or at the end.  E.g. for C<shard_digits> of "2" and
C<filename> "db.sqlite3", the shards would be "db_00.slite3", "db_01.sqlite3",
and so on.

=for Pod::Coverage::TrustPod store extract delete iterator initialize

=head1 AUTHORS

=over 4

=item *

David Golden <[e-mail]>

=item *

Leon Brocard <[e-mail]>

=back

=head1 COPYRIGHT AND LICENSE

This software is Copyright (c) 2011 by David Golden.

This is free software, licensed under:

  The Apache License, Version 2.0, January 2004

=cut