File Coverage

blib/lib/MogileFS/Store/SQLite.pm
Criterion Covered Total %
statement 13 15 86.6
branch n/a
condition n/a
subroutine 5 5 100.0
pod n/a
total 18 20 90.0


line stmt bran cond sub pod time code
1             package MogileFS::Store::SQLite;
2 1     1   3 use strict;
  1         2  
  1         33  
3 1     1   3 use warnings;
  1         3  
  1         35  
4 1     1   5 use DBI qw(:sql_types);
  1         1  
  1         416  
5 1     1   5 use Digest::MD5 qw(md5); # Used for lockid
  1         2  
  1         48  
6 1     1   192 use DBD::SQLite 1.13;
  0            
  0            
7             use MogileFS::Util qw(throw);
8             use base 'MogileFS::Store';
9             use File::Temp ();
10              
11             # --------------------------------------------------------------------------
12             # Package methods we override
13             # --------------------------------------------------------------------------
14              
15             sub post_dbi_connect {
16             my $self = shift;
17             $self->{dbh}->func(5000, 'busy_timeout');
18             $self->{lock_depth} = 0;
19             }
20              
21             sub want_raise_errors { 1 }
22              
23             sub dsn_of_dbhost {
24             my ($class, $dbname, $host) = @_;
25             return "DBI:SQLite:$dbname";
26             }
27              
28             sub dsn_of_root {
29             my ($class, $dbname, $host) = @_;
30             return "DBI:SQLite:$dbname";
31             }
32              
33             sub can_replace { 1 }
34             sub can_insertignore { 0 }
35             sub can_for_update { 0 }
36             sub unix_timestamp { "strftime('%s','now')" }
37              
38             sub init {
39             my $self = shift;
40             $self->SUPER::init;
41             $self->{lock_depth} = 0;
42             }
43              
44             # DBD::SQLite doesn't really have any table meta info methods
45             # And PRAGMA table_info() does not return "real" rows
46             sub column_type {
47             my ($self, $table, $col) = @_;
48             my $sth = $self->dbh->prepare("PRAGMA table_info($table)");
49             $sth->execute;
50             while (my $rec = $sth->fetchrow_arrayref) {
51             if ($rec->[1] eq $col) {
52             $sth->finish;
53             return $rec->[2];
54             }
55             }
56             return undef;
57             }
58              
59             sub lockid {
60             my ($lockname) = @_;
61             croak("Called with empty lockname! $lockname") unless (defined $lockname && length($lockname) > 0);
62             my $num = unpack 'N',md5($lockname);
63             return ($num & 0x7fffffff);
64             }
65              
66             # returns 1 if the lock holder is still alive, 0 if lock holder died
67             sub lock_holder_alive {
68             my ($self, $lockid, $lockname) = @_;
69             my $max_age = 3600;
70             my $force_unlock;
71              
72             my $dbh = $self->dbh;
73             my ($hostname, $pid, $acquiredat) = $dbh->selectrow_array('SELECT hostname,pid,acquiredat FROM lock WHERE lockid = ?', undef, $lockid);
74              
75             # maybe the lock was _just_ released
76             return 0 unless defined $pid;
77              
78             # if the lock is too old, don't check anything else
79             if (($acquiredat + $max_age) < time) {
80             $force_unlock = 1;
81             } elsif ($hostname eq MogileFS::Config->hostname) {
82             # maybe we were unlucky and the PID got recycled
83             if ($pid == $$) {
84             die("Possible lock recursion inside DB but not process (grabbing $lockname ($lockid, acquiredat=$acquiredat)");
85             }
86              
87             # don't force the lock if the process is still alive
88             return 1 if kill(0, $pid);
89              
90             $force_unlock = 1;
91             }
92              
93             return 0 unless $force_unlock;
94              
95             # lock holder is dead or the lock is too old: kill the lock
96             my $rv = $self->retry_on_deadlock(sub {
97             $dbh->do('DELETE FROM lock WHERE lockid = ? AND pid = ? AND hostname = ?', undef, $lockid, $pid, $hostname);
98             });
99              
100             # delete can fail if another process just deleted and regrabbed this lock
101             return $rv ? 0 : 1;
102             }
103              
104             # attempt to grab a lock of lockname, and timeout after timeout seconds.
105             # the lock should be unique in the space of (lockid). We can also detect
106             # if pid is dead as SQLite only runs on one host.
107             # returns 1 on success and 0 on timeout
108             sub get_lock {
109             my ($self, $lockname, $timeout) = @_;
110             my $lockid = lockid($lockname);
111             die "Lock recursion detected (grabbing $lockname ($lockid), had $self->{last_lock} (".lockid($self->{last_lock})."). Bailing out." if $self->{lock_depth};
112              
113             debug("$$ Locking $lockname ($lockid)\n") if $Mgd::DEBUG >= 5;
114             my $dbh = $self->dbh;
115             my $lock = undef;
116             my $try = sub {
117             $dbh->do('INSERT INTO lock (lockid,hostname,pid,acquiredat) VALUES (?, ?, ?, '.$self->unix_timestamp().')', undef, $lockid, MogileFS::Config->hostname, $$);
118             };
119              
120             while ($timeout >= 0 and not defined($lock)) {
121             $lock = eval { $self->retry_on_deadlock($try) };
122             if ($self->was_duplicate_error) {
123             # retry immediately if the lock holder died
124             if ($self->lock_holder_alive($lockid, $lockname)) {
125             sleep 1 if $timeout > 0;
126             $timeout--;
127             }
128             next;
129             }
130             $self->condthrow;
131             if (defined $lock and $lock == 1) {
132             $self->{lock_depth} = 1;
133             $self->{last_lock} = $lockname;
134             } else {
135             die "Something went horribly wrong while getting lock $lockname";
136             }
137             }
138             return $lock;
139             }
140              
141             # attempt to release a lock of lockname.
142             # returns 1 on success and 0 if no lock we have has that name.
143             sub release_lock {
144             my ($self, $lockname) = @_;
145             my $lockid = lockid($lockname);
146             debug("$$ Unlocking $lockname ($lockid)\n") if $Mgd::DEBUG >= 5;
147             my $rv = $self->retry_on_deadlock(sub {
148             $self->dbh->do('DELETE FROM lock WHERE lockid=? AND pid=? AND hostname=?', undef, $lockid, $$, MogileFS::Config->hostname);
149             });
150             debug("Double-release of lock $lockname!") if $self->{lock_depth} != 0 and $rv == 0 and $Mgd::DEBUG >= 2;
151             $self->condthrow;
152             $self->{lock_depth} = 0;
153             return $rv;
154             }
155              
156             # --------------------------------------------------------------------------
157             # Store-related things we override
158             # --------------------------------------------------------------------------
159              
160             # from sqlite3.h:
161             use constant SQLITE_BUSY => 5; # The database file is locked
162             use constant SQLITE_LOCKED => 6; # A table in the database is locked
163              
164             sub was_deadlock_error {
165             my $err = $_[0]->dbh->err or return 0;
166              
167             ($err == SQLITE_BUSY || $err == SQLITE_LOCKED);
168             }
169              
170             sub was_duplicate_error {
171             my $self = shift;
172             my $dbh = $self->dbh;
173             return 0 unless $dbh->err;
174             my $errstr = $dbh->errstr;
175             return 1 if $errstr =~ /(?:is|are) not unique/i;
176             return 1 if $errstr =~ /must be unique/i;
177             return 1 if $errstr =~ /UNIQUE constraint failed/i;
178             return 0;
179             }
180              
181             # --------------------------------------------------------------------------
182             # Test suite things we override
183             # --------------------------------------------------------------------------
184              
185             sub new_temp {
186             my ($fh, $filename) = File::Temp::tempfile();
187             close($fh);
188              
189             system("$FindBin::Bin/../mogdbsetup", "--type=SQLite", "--yes", "--dbname=$filename")
190             and die "Failed to run mogdbsetup ($FindBin::Bin/../mogdbsetup).";
191              
192             return MogileFS::Store->new_from_dsn_user_pass("DBI:SQLite:$filename",
193             "", "");
194             }
195              
196             sub table_exists {
197             my ($self, $table) = @_;
198             return eval {
199             my $sth = $self->dbh->prepare("EXPLAIN SELECT * FROM $table");
200             $sth->execute;
201             my $rec = $sth->fetchrow_hashref;
202             return $rec ? 1 : 0;
203             };
204             }
205              
206             sub setup_database {
207             my $self = shift;
208             # old installations may not have this, add this without changing
209             # schema version globally (unless the table itself changes)
210             $self->add_extra_tables('lock');
211             $self->create_table('lock');
212             return $self->SUPER::setup_database;
213             }
214              
215             # --------------------------------------------------------------------------
216             # Schema
217             # --------------------------------------------------------------------------
218              
219             sub TABLE_class {
220             "CREATE TABLE class (
221             dmid SMALLINT UNSIGNED NOT NULL,
222             classid TINYINT UNSIGNED NOT NULL,
223             classname VARCHAR(50),
224             mindevcount TINYINT UNSIGNED NOT NULL,
225             hashtype TINYINT UNSIGNED,
226             UNIQUE (dmid,classid),
227             UNIQUE (dmid,classname)
228             )"
229             }
230              
231             sub TABLE_file {
232             "CREATE TABLE file (
233             fid INT UNSIGNED NOT NULL PRIMARY KEY,
234             dmid SMALLINT UNSIGNED NOT NULL,
235             dkey VARCHAR(255),
236             length INT UNSIGNED,
237             classid TINYINT UNSIGNED NOT NULL,
238             devcount TINYINT UNSIGNED NOT NULL,
239             UNIQUE (dmid, dkey)
240             )"
241             }
242              
243             sub TABLE_device {
244             "CREATE TABLE device (
245             devid MEDIUMINT UNSIGNED NOT NULL,
246             hostid MEDIUMINT UNSIGNED NOT NULL,
247              
248             status ENUM('alive','dead','down','readonly','drain'),
249             weight MEDIUMINT DEFAULT 100,
250              
251             mb_total INT UNSIGNED,
252             mb_used INT UNSIGNED,
253             mb_asof INT UNSIGNED,
254             PRIMARY KEY (devid),
255             INDEX (status)
256             )"
257             }
258              
259             sub TABLE_tempfile {
260             "CREATE TABLE tempfile (
261             fid INTEGER PRIMARY KEY AUTOINCREMENT,
262             createtime INT UNSIGNED NOT NULL,
263             classid TINYINT UNSIGNED NOT NULL,
264             dmid SMALLINT UNSIGNED NOT NULL,
265             dkey VARCHAR(255),
266             devids VARCHAR(60)
267             )"
268             }
269              
270             sub TABLE_unreachable_fids {
271             "CREATE TABLE unreachable_fids (
272             fid INT UNSIGNED NOT NULL,
273             lastupdate INT UNSIGNED NOT NULL,
274             PRIMARY KEY (fid)
275             )"
276             }
277              
278             sub INDEXES_unreachable_fids {
279             ("CREATE INDEX lastupdate ON unreachable_fids (lastupdate)");
280             }
281              
282             sub TABLE_file_on {
283             "CREATE TABLE file_on (
284             fid INT UNSIGNED NOT NULL,
285             devid MEDIUMINT UNSIGNED NOT NULL,
286             PRIMARY KEY (fid, devid)
287             )"
288             }
289              
290             sub TABLE_fsck_log {
291             "CREATE TABLE fsck_log (
292             logid INTEGER PRIMARY KEY AUTOINCREMENT,
293             utime INT UNSIGNED NOT NULL,
294             fid INT UNSIGNED NULL,
295             evcode CHAR(4),
296             devid MEDIUMINT UNSIGNED
297             )"
298             }
299              
300             sub INDEXES_file_on {
301             ("CREATE INDEX devid ON file_on (devid)");
302             }
303              
304             sub INDEXES_device {
305             ("CREATE INDEX status ON device (status)");
306             }
307              
308             sub INDEXES_file_to_replicate {
309             ("CREATE INDEX nexttry ON file_to_replicate (nexttry)");
310             }
311              
312             sub INDEXES_file_to_delete_later {
313             ("CREATE INDEX delafter ON file_to_delete_later (delafter)");
314             }
315              
316             sub INDEXES_fsck_log {
317             ("CREATE INDEX utime ON fsck_log (utime)");
318             }
319              
320             sub INDEXES_file_to_queue {
321             ("CREATE INDEX type_nexttry ON file_to_queue (type,nexttry)");
322             }
323             sub INDEXES_file_to_delete2 {
324             ("CREATE INDEX file_to_delete2_nexttry ON file_to_delete2 (nexttry)");
325             }
326              
327             # Extra table
328             sub TABLE_lock {
329             "CREATE TABLE lock (
330             lockid INT UNSIGNED NOT NULL PRIMARY KEY,
331             hostname VARCHAR(255) NOT NULL,
332             pid INT UNSIGNED NOT NULL,
333             acquiredat INT UNSIGNED NOT NULL
334             )"
335             }
336              
337             sub filter_create_sql {
338             my ($self, $sql) = @_;
339             $sql =~ s/\bENUM\(.+?\)/TEXT/g;
340              
341             my ($table) = $sql =~ /create\s+table\s+(\S+)/i;
342             die "didn't find table" unless $table;
343             if ($self->can("INDEXES_$table")) {
344             $sql =~ s!,\s+INDEX\s+(\w+\s+)?\(.+?\)!!mg;
345             }
346              
347             return $sql;
348             }
349              
350             # eh. this is really atomic at all, but a) this is a demo db module,
351             # nobody should use SQLite in production, b) this method is going
352             # away, c) everything in SQLite is pretty atomic anyway with the
353             # db-level locks, d) the devcount field is no longer used. so i'm not
354             # caring much about doing this correctly.
355             sub update_devcount_atomic {
356             my ($self, $fidid) = @_;
357             $self->update_devcount($fidid);
358             }
359              
360             # SQLite is just for testing, so don't upgrade
361             sub upgrade_add_device_drain {
362             return 1;
363             }
364             sub upgrade_modify_server_settings_value { 1 }
365             sub upgrade_add_file_to_queue_arg { 1 }
366             sub upgrade_modify_device_size { 1 }
367             sub upgrade_add_host_readonly { 1 }
368              
369             sub BLOB_BIND_TYPE { SQL_BLOB }
370              
371             sub get_keys_like_operator {
372             my $self = shift;
373             my $bool = MogileFS::Config->server_setting_cached('case_sensitive_list_keys');
374              
375             # this is a dbh-wide change, but this is the only place we use LIKE
376             $self->dbh->do("PRAGMA case_sensitive_like = " . ($bool ? "ON" : "OFF"));
377             return "LIKE";
378             }
379              
380             1;
381              
382             __END__