2 * Force a readahead of files by opening them and reading the first bytes
4 * Copyright (C) Volker Lendecke 2008
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #include "system/filesys.h"
23 #include "smbd/smbd.h"
24 #include "lib/util/sys_rw.h"
25 #include "lib/util/sys_rw_data.h"
26 #include "lib/util/smb_strtox.h"
27 #include "lib/util_matching.h"
28 #include "lib/global_contexts.h"
30 static int vfs_preopen_debug_level
= DBGC_VFS
;
33 #define DBGC_CLASS vfs_preopen_debug_level
35 #define PREOPEN_MAX_DIGITS 19
36 #define PREOPEN_MAX_NUMBER (uint64_t)9999999999999999999ULL
40 struct preopen_helper
{
41 struct preopen_state
*state
;
42 struct tevent_fd
*fde
;
48 struct preopen_state
{
50 struct preopen_helper
*helpers
;
52 size_t to_read
; /* How many bytes to read in children? */
55 int queue_dbglvl
; /* DBGLVL_DEBUG by default */
56 int nomatch_dbglvl
; /* DBGLVL_INFO by default */
57 int match_dbglvl
; /* DBGLVL_INFO by default */
58 int reset_dbglvl
; /* DBGLVL_INFO by default */
59 int nodigits_dbglvl
; /* DBGLVL_WARNING by default */
60 int founddigits_dbglvl
; /* DBGLVL_NOTICE by default */
61 int push_dbglvl
; /* DBGLVL_NOTICE by default */
63 char *template_fname
; /* Filename to be sent to children */
64 size_t number_start
; /* start offset into "template_fname" */
65 int num_digits
; /* How many digits is the number long? */
67 uint64_t fnum_sent
; /* last fname sent to children */
69 uint64_t fnum_queue_end
;/* last fname to be sent, based on
70 * last open call + preopen:queuelen
73 struct samba_path_matching
*preopen_names
;
74 ssize_t last_match_idx
; /* remember the last match */
77 static void preopen_helper_destroy(struct preopen_helper
*c
)
83 kill(c
->pid
, SIGKILL
);
84 waitpid(c
->pid
, &status
, 0);
88 static void preopen_queue_run(struct preopen_state
*state
)
93 DBG_PREFIX(state
->queue_dbglvl
, ("START: "
94 "last_fname[%s] start_offset=%zu num_digits=%d "
95 "last_pushed_num=%"PRIu64
" queue_end_num=%"PRIu64
" num_helpers=%d\n",
96 state
->template_fname
,
100 state
->fnum_queue_end
,
101 state
->num_helpers
));
103 pdelimiter
= state
->template_fname
+ state
->number_start
105 delimiter
= *pdelimiter
;
107 while (state
->fnum_sent
< state
->fnum_queue_end
) {
113 for (helper
=0; helper
<state
->num_helpers
; helper
++) {
114 if (state
->helpers
[helper
].busy
) {
119 if (helper
== state
->num_helpers
) {
120 /* everyone is busy */
121 DBG_PREFIX(state
->queue_dbglvl
, ("BUSY: "
122 "template_fname[%s] start_offset=%zu num_digits=%d "
123 "last_pushed_num=%"PRIu64
" queue_end_num=%"PRIu64
"\n",
124 state
->template_fname
,
128 state
->fnum_queue_end
));
132 snprintf(state
->template_fname
+ state
->number_start
,
133 state
->num_digits
+ 1,
134 "%.*llu", state
->num_digits
,
135 (long long unsigned int)(state
->fnum_sent
+ 1));
136 *pdelimiter
= delimiter
;
138 DBG_PREFIX(state
->push_dbglvl
, (
139 "PUSH: fullpath[%s] to helper(idx=%d)\n",
140 state
->template_fname
, helper
));
142 to_write
= talloc_get_size(state
->template_fname
);
143 written
= write_data(state
->helpers
[helper
].fd
,
144 state
->template_fname
, to_write
);
145 state
->helpers
[helper
].busy
= true;
147 if (written
!= to_write
) {
148 preopen_helper_destroy(&state
->helpers
[helper
]);
150 state
->fnum_sent
+= 1;
152 DBG_PREFIX(state
->queue_dbglvl
, ("END: "
153 "template_fname[%s] start_offset=%zu num_digits=%d "
154 "last_pushed_num=%"PRIu64
" queue_end_num=%"PRIu64
"\n",
155 state
->template_fname
,
159 state
->fnum_queue_end
));
162 static void preopen_helper_readable(struct tevent_context
*ev
,
163 struct tevent_fd
*fde
, uint16_t flags
,
166 struct preopen_helper
*helper
= (struct preopen_helper
*)priv
;
167 struct preopen_state
*state
= helper
->state
;
171 if ((flags
& TEVENT_FD_READ
) == 0) {
175 nread
= read(helper
->fd
, &c
, 1);
177 preopen_helper_destroy(helper
);
181 helper
->busy
= false;
183 DBG_PREFIX(state
->queue_dbglvl
, ("BEFORE: preopen_queue_run\n"));
184 preopen_queue_run(state
);
185 DBG_PREFIX(state
->queue_dbglvl
, ("AFTER: preopen_queue_run\n"));
188 static int preopen_helpers_destructor(struct preopen_state
*c
)
192 for (i
=0; i
<c
->num_helpers
; i
++) {
193 if (c
->helpers
[i
].fd
== -1) {
196 preopen_helper_destroy(&c
->helpers
[i
]);
202 static bool preopen_helper_open_one(int sock_fd
, char **pnamebuf
,
203 size_t to_read
, void *filebuf
)
205 char *namebuf
= *pnamebuf
;
215 thistime
= read(sock_fd
, namebuf
+ nread
,
216 talloc_get_size(namebuf
) - nread
);
223 if (nread
== talloc_get_size(namebuf
)) {
224 namebuf
= talloc_realloc(
226 talloc_get_size(namebuf
) * 2);
227 if (namebuf
== NULL
) {
232 } while (namebuf
[nread
- 1] != '\0');
234 fd
= open(namebuf
, O_RDONLY
);
238 nread
= read(fd
, filebuf
, to_read
);
242 sys_write_v(sock_fd
, &c
, 1);
246 static bool preopen_helper(int fd
, size_t to_read
)
251 namebuf
= talloc_array(NULL
, char, 1024);
252 if (namebuf
== NULL
) {
256 readbuf
= talloc_size(NULL
, to_read
);
257 if (readbuf
== NULL
) {
258 TALLOC_FREE(namebuf
);
262 while (preopen_helper_open_one(fd
, &namebuf
, to_read
, readbuf
)) {
266 TALLOC_FREE(readbuf
);
267 TALLOC_FREE(namebuf
);
271 static NTSTATUS
preopen_init_helper(struct preopen_helper
*h
)
276 if (socketpair(AF_UNIX
, SOCK_STREAM
, 0, fdpair
) == -1) {
277 status
= map_nt_error_from_unix(errno
);
278 DEBUG(10, ("socketpair() failed: %s\n", strerror(errno
)));
285 return map_nt_error_from_unix(errno
);
290 preopen_helper(fdpair
[1], h
->state
->to_read
);
295 h
->fde
= tevent_add_fd(global_event_context(), h
->state
, h
->fd
,
296 TEVENT_FD_READ
, preopen_helper_readable
, h
);
297 if (h
->fde
== NULL
) {
300 return NT_STATUS_NO_MEMORY
;
306 static NTSTATUS
preopen_init_helpers(TALLOC_CTX
*mem_ctx
, size_t to_read
,
307 int num_helpers
, int queue_max
,
308 struct preopen_state
**presult
)
310 struct preopen_state
*result
;
313 result
= talloc(mem_ctx
, struct preopen_state
);
314 if (result
== NULL
) {
315 return NT_STATUS_NO_MEMORY
;
318 result
->num_helpers
= num_helpers
;
319 result
->helpers
= talloc_array(result
, struct preopen_helper
,
321 if (result
->helpers
== NULL
) {
323 return NT_STATUS_NO_MEMORY
;
326 result
->to_read
= to_read
;
327 result
->queue_max
= queue_max
;
328 result
->template_fname
= NULL
;
329 result
->fnum_sent
= 0;
330 result
->fnum_queue_end
= 0;
332 for (i
=0; i
<num_helpers
; i
++) {
333 result
->helpers
[i
].state
= result
;
334 result
->helpers
[i
].fd
= -1;
337 talloc_set_destructor(result
, preopen_helpers_destructor
);
339 for (i
=0; i
<num_helpers
; i
++) {
340 preopen_init_helper(&result
->helpers
[i
]);
347 static void preopen_free_helpers(void **ptr
)
352 static struct preopen_state
*preopen_state_get(vfs_handle_struct
*handle
)
354 struct preopen_state
*state
;
356 const char *namelist
;
358 if (SMB_VFS_HANDLE_TEST_DATA(handle
)) {
359 SMB_VFS_HANDLE_GET_DATA(handle
, state
, struct preopen_state
,
364 namelist
= lp_parm_const_string(SNUM(handle
->conn
), "preopen", "names",
367 if (namelist
== NULL
) {
371 status
= preopen_init_helpers(
373 lp_parm_int(SNUM(handle
->conn
), "preopen", "num_bytes", 1),
374 lp_parm_int(SNUM(handle
->conn
), "preopen", "helpers", 1),
375 lp_parm_int(SNUM(handle
->conn
), "preopen", "queuelen", 10),
377 if (!NT_STATUS_IS_OK(status
)) {
381 state
->queue_dbglvl
= lp_parm_int(SNUM(handle
->conn
), "preopen", "queue_log_level", DBGLVL_DEBUG
);
382 state
->nomatch_dbglvl
= lp_parm_int(SNUM(handle
->conn
), "preopen", "nomatch_log_level", DBGLVL_INFO
);
383 state
->match_dbglvl
= lp_parm_int(SNUM(handle
->conn
), "preopen", "match_log_level", DBGLVL_INFO
);
384 state
->reset_dbglvl
= lp_parm_int(SNUM(handle
->conn
), "preopen", "reset_log_level", DBGLVL_INFO
);
385 state
->nodigits_dbglvl
= lp_parm_int(SNUM(handle
->conn
), "preopen", "nodigits_log_level", DBGLVL_WARNING
);
386 state
->founddigits_dbglvl
= lp_parm_int(SNUM(handle
->conn
), "preopen", "founddigits_log_level", DBGLVL_NOTICE
);
387 state
->push_dbglvl
= lp_parm_int(SNUM(handle
->conn
), "preopen", "push_log_level", DBGLVL_NOTICE
);
389 if (lp_parm_bool(SNUM(handle
->conn
), "preopen", "posix-basic-regex", false)) {
390 status
= samba_path_matching_regex_sub1_create(state
,
392 &state
->preopen_names
);
394 status
= samba_path_matching_mswild_create(state
,
395 true, /* case_sensitive */
397 &state
->preopen_names
);
399 if (!NT_STATUS_IS_OK(status
)) {
403 state
->last_match_idx
= -1;
405 if (!SMB_VFS_HANDLE_TEST_DATA(handle
)) {
406 SMB_VFS_HANDLE_SET_DATA(handle
, state
, preopen_free_helpers
,
407 struct preopen_state
, return NULL
);
413 static bool preopen_parse_fname(const char *fname
, uint64_t *pnum
,
414 size_t *pstart_idx
, int *pnum_digits
)
416 char digits
[PREOPEN_MAX_DIGITS
+1] = { 0, };
419 unsigned long long num
;
420 size_t start_idx
= 0;
424 if (*pstart_idx
> 0 && *pnum_digits
> 0) {
426 * If the caller knowns
427 * how many digits are expected
428 * and on what position,
429 * we should copy the exact
430 * subset before we start
431 * parsing the string into a number
434 if (*pnum_digits
> PREOPEN_MAX_DIGITS
) {
436 * a string with as much digits as
437 * PREOPEN_MAX_DIGITS is the longest
438 * string that would make any sense for us.
440 * The rest will be checked via
445 p
= fname
+ *pstart_idx
;
446 memcpy(digits
, p
, *pnum_digits
);
448 start_idx
= *pstart_idx
;
452 p
= strrchr_m(fname
, '/');
458 while (p
[0] != '\0') {
459 if (isdigit(p
[0]) && isdigit(p
[1]) && isdigit(p
[2])) {
465 /* no digits around */
469 start_idx
= (p
- fname
);
472 num
= smb_strtoull(p
, (char **)&q
, 10, &error
, SMB_STR_STANDARD
);
477 if (num
>= PREOPEN_MAX_NUMBER
) {
482 num_digits
= (q
- p
);
484 if (*pnum_digits
!= -1 && *pnum_digits
!= num_digits
) {
486 * If the caller knowns how many digits
487 * it expects we should fail if we got something
494 *pstart_idx
= start_idx
;
495 *pnum_digits
= num_digits
;
499 static uint64_t num_digits_max_value(int num_digits
)
501 uint64_t num_max
= 1;
504 if (num_digits
< 1) {
507 if (num_digits
>= PREOPEN_MAX_DIGITS
) {
508 return PREOPEN_MAX_NUMBER
;
511 for (i
= 0; i
< num_digits
; i
++) {
519 * 999 instead of 1000
524 static int preopen_openat(struct vfs_handle_struct
*handle
,
525 const struct files_struct
*dirfsp
,
526 const struct smb_filename
*smb_fname
,
527 struct files_struct
*fsp
,
528 const struct vfs_open_how
*how
)
530 const char *dirname
= dirfsp
->fsp_name
->base_name
;
531 struct preopen_state
*state
;
536 char *new_template
= NULL
;
537 size_t new_start
= 0;
540 ssize_t match_idx
= -1;
541 ssize_t replace_start
= -1;
542 ssize_t replace_end
= -1;
543 bool need_reset
= false;
545 DBG_DEBUG("called on %s\n", smb_fname_str_dbg(smb_fname
));
547 state
= preopen_state_get(handle
);
549 return SMB_VFS_NEXT_OPENAT(handle
,
556 res
= SMB_VFS_NEXT_OPENAT(handle
, dirfsp
, smb_fname
, fsp
, how
);
561 if ((how
->flags
& O_ACCMODE
) != O_RDONLY
) {
566 * Make sure we can later construct an absolute pathname
568 if (dirname
[0] != '/') {
572 * There's no point in preopen the directory itself.
574 if (ISDOT(smb_fname
->base_name
)) {
578 * If we got an absolute path in
579 * smb_fname it's most likely the
580 * reopen via /proc/self/fd/$fd
582 if (smb_fname
->base_name
[0] == '/') {
586 status
= samba_path_matching_check_last_component(state
->preopen_names
,
587 smb_fname
->base_name
,
591 if (!NT_STATUS_IS_OK(status
)) {
595 DBG_PREFIX(state
->nomatch_dbglvl
, (
596 "No match with the preopen:names list by name[%s]\n",
597 smb_fname_str_dbg(smb_fname
)));
601 if (replace_start
!= -1 && replace_end
!= -1) {
602 DBG_PREFIX(state
->match_dbglvl
, (
603 "Pattern(idx=%zd) from preopen:names list matched name[%s] hints(start=%zd,end=%zd)\n",
604 match_idx
, smb_fname_str_dbg(smb_fname
), replace_start
, replace_end
));
606 DBG_PREFIX(state
->match_dbglvl
, (
607 "Pattern(idx=%zd) from preopen:names list matched name[%s]\n",
608 match_idx
, smb_fname_str_dbg(smb_fname
)));
611 new_template
= talloc_asprintf(
613 dirname
, smb_fname
->base_name
);
614 if (new_template
== NULL
) {
615 DBG_ERR("talloc_asprintf(%s/%s) failed\n",
616 dirname
, smb_fname_str_dbg(smb_fname
));
620 if (replace_start
!= -1 && replace_end
!= -1) {
621 size_t dirofs
= strlen(dirname
) + 1;
622 new_start
= dirofs
+ replace_start
;
623 new_digits
= replace_end
- replace_start
;
626 if (!preopen_parse_fname(new_template
, &num
,
627 &new_start
, &new_digits
)) {
628 DBG_PREFIX(state
->nodigits_dbglvl
, (
629 "Pattern(idx=%zd) no valid digits found on fullpath[%s]\n",
630 match_idx
, new_template
));
631 TALLOC_FREE(new_template
);
634 new_end
= new_start
+ new_digits
;
636 DBG_PREFIX(state
->founddigits_dbglvl
, (
637 "Pattern(idx=%zd) found num_digits[%d] start_offset[%zd] parsed_num[%"PRIu64
"] fullpath[%s]\n",
638 match_idx
, new_digits
, new_start
, num
, new_template
));
640 if (state
->last_match_idx
!= match_idx
) {
642 * If a different pattern caused the match
643 * we better reset the queue
645 if (state
->last_match_idx
!= -1) {
646 DBG_PREFIX(state
->reset_dbglvl
, ("RESET: "
647 "pattern changed from idx=%zd to idx=%zd by fullpath[%s]\n",
648 state
->last_match_idx
, match_idx
, new_template
));
651 } else if (state
->number_start
!= new_start
) {
653 * If the digits started at a different position
654 * we better reset the queue
656 DBG_PREFIX(state
->reset_dbglvl
, ("RESET: "
657 "start_offset changed from byte=%zd to byte=%zd by fullpath[%s]\n",
658 state
->number_start
, new_start
, new_template
));
660 } else if (state
->num_digits
!= new_digits
) {
662 * If number of digits changed
663 * we better reset the queue
665 DBG_PREFIX(state
->reset_dbglvl
, ("RESET: "
666 "num_digits changed %d to %d by fullpath[%s]\n",
667 state
->num_digits
, new_digits
, new_template
));
669 } else if (strncmp(state
->template_fname
, new_template
, new_start
) != 0) {
671 * If name before the digits changed
672 * we better reset the queue
674 DBG_PREFIX(state
->reset_dbglvl
, ("RESET: "
675 "leading pathprefix[%.*s] changed by fullpath[%s]\n",
676 (int)state
->number_start
, state
->template_fname
, new_template
));
678 } else if (strcmp(state
->template_fname
+ new_end
, new_template
+ new_end
) != 0) {
680 * If name after the digits changed
681 * we better reset the queue
683 DBG_PREFIX(state
->reset_dbglvl
, ("RESET: "
684 "trailing suffix[%s] changed by fullpath[%s]\n",
685 state
->template_fname
+ new_end
, new_template
));
693 state
->fnum_sent
= 0;
694 state
->fnum_queue_end
= 0;
695 state
->last_match_idx
= match_idx
;
698 TALLOC_FREE(state
->template_fname
);
699 state
->template_fname
= new_template
;
700 state
->number_start
= new_start
;
701 state
->num_digits
= new_digits
;
703 if (num
> state
->fnum_sent
) {
705 * Helpers were too slow, there's no point in reading
706 * files in helpers that we already read in the
709 state
->fnum_sent
= num
;
712 if ((state
->fnum_queue_end
!= 0) /* Something was started earlier */
713 && (num
< (state
->fnum_queue_end
- state
->queue_max
))) {
715 * "num" is before the queue we announced. This means
716 * a new run is started.
718 state
->fnum_sent
= num
;
721 num_max
= num_digits_max_value(state
->num_digits
);
722 state
->fnum_queue_end
= MIN(num_max
, num
+ state
->queue_max
);
724 DBG_PREFIX(state
->queue_dbglvl
, ("BEFORE: preopen_queue_run\n"));
725 preopen_queue_run(state
);
726 DBG_PREFIX(state
->queue_dbglvl
, ("AFTER: preopen_queue_run\n"));
731 static struct vfs_fn_pointers vfs_preopen_fns
= {
732 .openat_fn
= preopen_openat
,
736 NTSTATUS
vfs_preopen_init(TALLOC_CTX
*ctx
)
740 status
= smb_register_vfs(SMB_VFS_INTERFACE_VERSION
,
743 if (!NT_STATUS_IS_OK(status
)) {
747 vfs_preopen_debug_level
= debug_add_class("preopen");
748 if (vfs_preopen_debug_level
== -1) {
749 vfs_preopen_debug_level
= DBGC_VFS
;
750 DBG_ERR("Couldn't register custom debugging class!\n");
752 DBG_DEBUG("Debug class number of 'preopen': %d\n",
753 vfs_preopen_debug_level
);