1 /* binlog.c - binary log implementation */
3 /* Copyright (C) 2008 Graham Barr
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
26 #include <sys/resource.h>
27 #include <sys/param.h>
38 /* max size we will create a log file */
39 size_t binlog_size_limit
= 10 << 20;
41 char *binlog_dir
= NULL
;
42 static int binlog_index
= 0;
43 static int binlog_fd
= -1;
44 static int binlog_version
= 1;
45 static size_t bytes_written
;
47 static binlog first_binlog
= NULL
, last_binlog
= NULL
;
60 dirp
= opendir(binlog_dir
);
63 while ((dp
= readdir(dirp
)) != NULL
) {
64 name_len
= strlen(dp
->d_name
);
65 if (name_len
> 7 && !strncmp("binlog.", dp
->d_name
, 7)) {
66 val
= strtol(dp
->d_name
+ 7, &endptr
, 10);
67 if (endptr
&& *endptr
== 0) {
68 if (max
== 0 || val
> max
) max
= val
;
69 if (min
== 0 || val
< min
) min
= val
;
75 binlog_index
= (int) max
;
82 binlog b
= first_binlog
;
86 first_binlog
= b
->next
;
87 if (!first_binlog
) last_binlog
= NULL
;
101 binlog_dref(binlog b
)
104 if (b
->refs
< 1) return twarnx("refs is zero for binlog: %s", b
->path
);
108 while (first_binlog
&& first_binlog
->refs
== 0) binlog_remove_first();
113 binlog_replay(int fd
, job binlog_jobs
)
118 char tubename
[MAX_TUBE_NAME_LEN
];
122 if (read(fd
, &version
, sizeof(version
)) < sizeof(version
)) {
123 return twarn("read()");
125 if (version
!= binlog_version
) {
126 return twarnx("binlog version mismatch %d %d", version
, binlog_version
);
129 while (read(fd
, &namelen
, sizeof(size_t)) == sizeof(size_t)) {
130 if (namelen
> 0 && read(fd
, tubename
, namelen
) != namelen
) {
131 return twarnx("oops %x %d", namelen
, (int)lseek(fd
, SEEK_CUR
, 0));
134 tubename
[namelen
] = '\0';
135 if (read(fd
, &js
, sizeof(struct job
)) != sizeof(struct job
)) {
136 return twarn("read()");
141 case JOB_STATE_INVALID
:
144 binlog_dref(j
->binlog
);
149 case JOB_STATE_READY
:
150 case JOB_STATE_DELAYED
:
152 t
= tube_find_or_make(tubename
);
153 j
= make_job_with_id(js
.pri
, js
.delay
, js
.ttr
, js
.body_size
,
155 j
->next
= j
->prev
= j
;
156 j
->creation
= js
.creation
;
157 job_insert(binlog_jobs
, j
);
158 if (read(fd
, j
->body
, js
.body_size
) < js
.body_size
) {
167 j
->deadline
= js
.deadline
;
171 j
->timeout_ct
= js
.timeout_ct
;
172 j
->release_ct
= js
.release_ct
;
173 j
->bury_ct
= js
.bury_ct
;
174 j
->kick_ct
= js
.kick_ct
;
176 /* this is a complete record, so we can move the binlog ref */
177 if (namelen
&& js
.body_size
) {
178 binlog_dref(j
->binlog
);
179 j
->binlog
= binlog_iref(last_binlog
);
188 if (binlog_fd
< 0) return;
190 binlog_dref(last_binlog
);
195 add_binlog(char *path
)
199 b
= (binlog
)malloc(sizeof(struct binlog
) + strlen(path
) + 1);
200 if (!b
) return twarnx("OOM"), NULL
;
201 strcpy(b
->path
, path
);
204 if (last_binlog
) last_binlog
->next
= b
;
206 if (!first_binlog
) first_binlog
= b
;
218 if (!binlog_dir
) return -1;
219 r
= snprintf(path
, PATH_MAX
, "%s/binlog.%d", binlog_dir
, ++binlog_index
);
220 if (r
> PATH_MAX
) return twarnx("path too long: %s", binlog_dir
), -1;
222 if (!binlog_iref(add_binlog(path
))) return -1;
223 fd
= open(path
, O_WRONLY
| O_CREAT
, 0400);
226 twarn("Cannot open binlog %s", path
);
231 bytes_written
= write(fd
, &binlog_version
, sizeof(int));
233 if (bytes_written
< sizeof(int)) {
234 twarn("Cannot write to binlog");
236 binlog_dref(last_binlog
);
246 if (binlog_fd
< 0) return;
248 binlog_dref(last_binlog
);
249 binlog_fd
= binlog_open();
253 binlog_write_job(job j
)
255 size_t tube_namelen
, to_write
;
256 struct iovec vec
[4], *vptr
;
259 if (binlog_fd
< 0) return;
262 vec
[0].iov_base
= (char *) &tube_namelen
;
263 vec
[0].iov_len
= sizeof(size_t);
264 to_write
= sizeof(size_t);
266 vec
[1].iov_base
= j
->tube
->name
;
269 /* we could save some bytes in the binlog file by only saving some parts of
271 vec
[2].iov_base
= (char *) j
;
272 vec
[2].iov_len
= sizeof(struct job
);
273 to_write
+= sizeof(struct job
);
275 if (j
->state
== JOB_STATE_READY
|| j
->state
== JOB_STATE_DELAYED
) {
277 tube_namelen
= strlen(j
->tube
->name
);
278 vec
[1].iov_len
= tube_namelen
;
279 to_write
+= tube_namelen
;
281 vec
[3].iov_base
= j
->body
;
282 vec
[3].iov_len
= j
->body_size
;
283 to_write
+= j
->body_size
;
285 } else if (j
->state
== JOB_STATE_INVALID
) {
286 if (j
->binlog
) binlog_dref(j
->binlog
);
289 return twarnx("unserializable job state: %d", j
->state
);
292 if ((bytes_written
+ to_write
) > binlog_size_limit
) binlog_open_next();
293 if (binlog_fd
< 0) return;
295 if (j
->state
&& !j
->binlog
) j
->binlog
= binlog_iref(last_binlog
);
297 while (to_write
> 0) {
298 size_t written
= writev(binlog_fd
, vec
, vcnt
);
301 twarn("Cannot write to binlog");
306 bytes_written
+= written
;
308 if (to_write
> 0 && written
> 0) {
309 for (vptr
= vec
; written
>= vptr
->iov_len
; vptr
++) {
310 written
-= vptr
->iov_len
;
313 vptr
->iov_base
= (char *) vptr
->iov_base
+ written
;
314 vptr
->iov_len
-= written
;
321 binlog_read(job binlog_jobs
)
323 int binlog_index_min
;
329 if (!binlog_dir
) return;
331 if (stat(binlog_dir
, &sbuf
) < 0) {
332 if (mkdir(binlog_dir
, 0700) < 0) return twarn("%s", binlog_dir
);
333 } else if (!(sbuf
.st_mode
& S_IFDIR
)) {
334 twarnx("%s", binlog_dir
);
338 binlog_index_min
= binlog_scan_dir();
340 if (binlog_index_min
) {
341 for (idx
= binlog_index_min
; idx
<= binlog_index
; idx
++) {
342 r
= snprintf(path
, PATH_MAX
, "%s/binlog.%d", binlog_dir
, idx
);
343 if (r
> PATH_MAX
) return twarnx("path too long: %s", binlog_dir
);
345 fd
= open(path
, O_RDONLY
);
350 b
= binlog_iref(add_binlog(path
));
351 binlog_replay(fd
, binlog_jobs
);
363 binlog_fd
= binlog_open();