explicitly mark deprecated functions as deprecated
[isl.git] / isl_schedule.c
blobfca395f964ae201ba48413a36de8bb3c683e0398
1 /*
2 * Copyright 2011 INRIA Saclay
3 * Copyright 2012-2013 Ecole Normale Superieure
5 * Use of this software is governed by the MIT license
7 * Written by Sven Verdoolaege, INRIA Saclay - Ile-de-France,
8 * Parc Club Orsay Universite, ZAC des vignes, 4 rue Jacques Monod,
9 * 91893 Orsay, France
10 * and Ecole Normale Superieure, 45 rue d'Ulm, 75230 Paris, France
13 #include <isl_ctx_private.h>
14 #include <isl_map_private.h>
15 #include <isl_space_private.h>
16 #include <isl_aff_private.h>
17 #include <isl/hash.h>
18 #include <isl/constraint.h>
19 #include <isl/schedule.h>
20 #include <isl_mat_private.h>
21 #include <isl_vec_private.h>
22 #include <isl/set.h>
23 #include <isl_seq.h>
24 #include <isl_tab.h>
25 #include <isl_dim_map.h>
26 #include <isl/map_to_basic_set.h>
27 #include <isl_sort.h>
28 #include <isl_schedule_private.h>
29 #include <isl_band_private.h>
30 #include <isl_options_private.h>
31 #include <isl_tarjan.h>
34 * The scheduling algorithm implemented in this file was inspired by
35 * Bondhugula et al., "Automatic Transformations for Communication-Minimized
36 * Parallelization and Locality Optimization in the Polyhedral Model".
39 __isl_give isl_schedule_constraints *isl_schedule_constraints_copy(
40 __isl_keep isl_schedule_constraints *sc)
42 isl_ctx *ctx;
43 isl_schedule_constraints *sc_copy;
44 enum isl_edge_type i;
46 ctx = isl_union_set_get_ctx(sc->domain);
47 sc_copy = isl_calloc_type(ctx, struct isl_schedule_constraints);
48 if (!sc_copy)
49 return NULL;
51 sc_copy->domain = isl_union_set_copy(sc->domain);
52 if (!sc_copy->domain)
53 return isl_schedule_constraints_free(sc_copy);
55 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
56 sc_copy->constraint[i] = isl_union_map_copy(sc->constraint[i]);
57 if (!sc_copy->constraint[i])
58 return isl_schedule_constraints_free(sc_copy);
61 return sc_copy;
65 /* Construct an isl_schedule_constraints object for computing a schedule
66 * on "domain". The initial object does not impose any constraints.
68 __isl_give isl_schedule_constraints *isl_schedule_constraints_on_domain(
69 __isl_take isl_union_set *domain)
71 isl_ctx *ctx;
72 isl_space *space;
73 isl_schedule_constraints *sc;
74 isl_union_map *empty;
75 enum isl_edge_type i;
77 if (!domain)
78 return NULL;
80 ctx = isl_union_set_get_ctx(domain);
81 sc = isl_calloc_type(ctx, struct isl_schedule_constraints);
82 if (!sc)
83 goto error;
85 space = isl_union_set_get_space(domain);
86 sc->domain = domain;
87 empty = isl_union_map_empty(space);
88 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
89 sc->constraint[i] = isl_union_map_copy(empty);
90 if (!sc->constraint[i])
91 sc->domain = isl_union_set_free(sc->domain);
93 isl_union_map_free(empty);
95 if (!sc->domain)
96 return isl_schedule_constraints_free(sc);
98 return sc;
99 error:
100 isl_union_set_free(domain);
101 return NULL;
104 /* Replace the validity constraints of "sc" by "validity".
106 __isl_give isl_schedule_constraints *isl_schedule_constraints_set_validity(
107 __isl_take isl_schedule_constraints *sc,
108 __isl_take isl_union_map *validity)
110 if (!sc || !validity)
111 goto error;
113 isl_union_map_free(sc->constraint[isl_edge_validity]);
114 sc->constraint[isl_edge_validity] = validity;
116 return sc;
117 error:
118 isl_schedule_constraints_free(sc);
119 isl_union_map_free(validity);
120 return NULL;
123 /* Replace the coincidence constraints of "sc" by "coincidence".
125 __isl_give isl_schedule_constraints *isl_schedule_constraints_set_coincidence(
126 __isl_take isl_schedule_constraints *sc,
127 __isl_take isl_union_map *coincidence)
129 if (!sc || !coincidence)
130 goto error;
132 isl_union_map_free(sc->constraint[isl_edge_coincidence]);
133 sc->constraint[isl_edge_coincidence] = coincidence;
135 return sc;
136 error:
137 isl_schedule_constraints_free(sc);
138 isl_union_map_free(coincidence);
139 return NULL;
142 /* Replace the proximity constraints of "sc" by "proximity".
144 __isl_give isl_schedule_constraints *isl_schedule_constraints_set_proximity(
145 __isl_take isl_schedule_constraints *sc,
146 __isl_take isl_union_map *proximity)
148 if (!sc || !proximity)
149 goto error;
151 isl_union_map_free(sc->constraint[isl_edge_proximity]);
152 sc->constraint[isl_edge_proximity] = proximity;
154 return sc;
155 error:
156 isl_schedule_constraints_free(sc);
157 isl_union_map_free(proximity);
158 return NULL;
161 /* Replace the conditional validity constraints of "sc" by "condition"
162 * and "validity".
164 __isl_give isl_schedule_constraints *
165 isl_schedule_constraints_set_conditional_validity(
166 __isl_take isl_schedule_constraints *sc,
167 __isl_take isl_union_map *condition,
168 __isl_take isl_union_map *validity)
170 if (!sc || !condition || !validity)
171 goto error;
173 isl_union_map_free(sc->constraint[isl_edge_condition]);
174 sc->constraint[isl_edge_condition] = condition;
175 isl_union_map_free(sc->constraint[isl_edge_conditional_validity]);
176 sc->constraint[isl_edge_conditional_validity] = validity;
178 return sc;
179 error:
180 isl_schedule_constraints_free(sc);
181 isl_union_map_free(condition);
182 isl_union_map_free(validity);
183 return NULL;
186 __isl_null isl_schedule_constraints *isl_schedule_constraints_free(
187 __isl_take isl_schedule_constraints *sc)
189 enum isl_edge_type i;
191 if (!sc)
192 return NULL;
194 isl_union_set_free(sc->domain);
195 for (i = isl_edge_first; i <= isl_edge_last; ++i)
196 isl_union_map_free(sc->constraint[i]);
198 free(sc);
200 return NULL;
203 isl_ctx *isl_schedule_constraints_get_ctx(
204 __isl_keep isl_schedule_constraints *sc)
206 return sc ? isl_union_set_get_ctx(sc->domain) : NULL;
209 void isl_schedule_constraints_dump(__isl_keep isl_schedule_constraints *sc)
211 if (!sc)
212 return;
214 fprintf(stderr, "domain: ");
215 isl_union_set_dump(sc->domain);
216 fprintf(stderr, "validity: ");
217 isl_union_map_dump(sc->constraint[isl_edge_validity]);
218 fprintf(stderr, "proximity: ");
219 isl_union_map_dump(sc->constraint[isl_edge_proximity]);
220 fprintf(stderr, "coincidence: ");
221 isl_union_map_dump(sc->constraint[isl_edge_coincidence]);
222 fprintf(stderr, "condition: ");
223 isl_union_map_dump(sc->constraint[isl_edge_condition]);
224 fprintf(stderr, "conditional_validity: ");
225 isl_union_map_dump(sc->constraint[isl_edge_conditional_validity]);
228 /* Align the parameters of the fields of "sc".
230 static __isl_give isl_schedule_constraints *
231 isl_schedule_constraints_align_params(__isl_take isl_schedule_constraints *sc)
233 isl_space *space;
234 enum isl_edge_type i;
236 if (!sc)
237 return NULL;
239 space = isl_union_set_get_space(sc->domain);
240 for (i = isl_edge_first; i <= isl_edge_last; ++i)
241 space = isl_space_align_params(space,
242 isl_union_map_get_space(sc->constraint[i]));
244 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
245 sc->constraint[i] = isl_union_map_align_params(
246 sc->constraint[i], isl_space_copy(space));
247 if (!sc->constraint[i])
248 space = isl_space_free(space);
250 sc->domain = isl_union_set_align_params(sc->domain, space);
251 if (!sc->domain)
252 return isl_schedule_constraints_free(sc);
254 return sc;
257 /* Return the total number of isl_maps in the constraints of "sc".
259 static __isl_give int isl_schedule_constraints_n_map(
260 __isl_keep isl_schedule_constraints *sc)
262 enum isl_edge_type i;
263 int n = 0;
265 for (i = isl_edge_first; i <= isl_edge_last; ++i)
266 n += isl_union_map_n_map(sc->constraint[i]);
268 return n;
271 /* Internal information about a node that is used during the construction
272 * of a schedule.
273 * dim represents the space in which the domain lives
274 * sched is a matrix representation of the schedule being constructed
275 * for this node
276 * sched_map is an isl_map representation of the same (partial) schedule
277 * sched_map may be NULL
278 * rank is the number of linearly independent rows in the linear part
279 * of sched
280 * the columns of cmap represent a change of basis for the schedule
281 * coefficients; the first rank columns span the linear part of
282 * the schedule rows
283 * cinv is the inverse of cmap.
284 * start is the first variable in the LP problem in the sequences that
285 * represents the schedule coefficients of this node
286 * nvar is the dimension of the domain
287 * nparam is the number of parameters or 0 if we are not constructing
288 * a parametric schedule
290 * scc is the index of SCC (or WCC) this node belongs to
292 * band contains the band index for each of the rows of the schedule.
293 * band_id is used to differentiate between separate bands at the same
294 * level within the same parent band, i.e., bands that are separated
295 * by the parent band or bands that are independent of each other.
296 * coincident contains a boolean for each of the rows of the schedule,
297 * indicating whether the corresponding scheduling dimension satisfies
298 * the coincidence constraints in the sense that the corresponding
299 * dependence distances are zero.
301 struct isl_sched_node {
302 isl_space *dim;
303 isl_mat *sched;
304 isl_map *sched_map;
305 int rank;
306 isl_mat *cmap;
307 isl_mat *cinv;
308 int start;
309 int nvar;
310 int nparam;
312 int scc;
314 int *band;
315 int *band_id;
316 int *coincident;
319 static int node_has_dim(const void *entry, const void *val)
321 struct isl_sched_node *node = (struct isl_sched_node *)entry;
322 isl_space *dim = (isl_space *)val;
324 return isl_space_is_equal(node->dim, dim);
327 /* An edge in the dependence graph. An edge may be used to
328 * ensure validity of the generated schedule, to minimize the dependence
329 * distance or both
331 * map is the dependence relation, with i -> j in the map if j depends on i
332 * tagged_condition and tagged_validity contain the union of all tagged
333 * condition or conditional validity dependence relations that
334 * specialize the dependence relation "map"; that is,
335 * if (i -> a) -> (j -> b) is an element of "tagged_condition"
336 * or "tagged_validity", then i -> j is an element of "map".
337 * If these fields are NULL, then they represent the empty relation.
338 * src is the source node
339 * dst is the sink node
340 * validity is set if the edge is used to ensure correctness
341 * coincidence is used to enforce zero dependence distances
342 * proximity is set if the edge is used to minimize dependence distances
343 * condition is set if the edge represents a condition
344 * for a conditional validity schedule constraint
345 * local can only be set for condition edges and indicates that
346 * the dependence distance over the edge should be zero
347 * conditional_validity is set if the edge is used to conditionally
348 * ensure correctness
350 * For validity edges, start and end mark the sequence of inequality
351 * constraints in the LP problem that encode the validity constraint
352 * corresponding to this edge.
354 struct isl_sched_edge {
355 isl_map *map;
356 isl_union_map *tagged_condition;
357 isl_union_map *tagged_validity;
359 struct isl_sched_node *src;
360 struct isl_sched_node *dst;
362 unsigned validity : 1;
363 unsigned coincidence : 1;
364 unsigned proximity : 1;
365 unsigned local : 1;
366 unsigned condition : 1;
367 unsigned conditional_validity : 1;
369 int start;
370 int end;
373 /* Internal information about the dependence graph used during
374 * the construction of the schedule.
376 * intra_hmap is a cache, mapping dependence relations to their dual,
377 * for dependences from a node to itself
378 * inter_hmap is a cache, mapping dependence relations to their dual,
379 * for dependences between distinct nodes
381 * n is the number of nodes
382 * node is the list of nodes
383 * maxvar is the maximal number of variables over all nodes
384 * max_row is the allocated number of rows in the schedule
385 * n_row is the current (maximal) number of linearly independent
386 * rows in the node schedules
387 * n_total_row is the current number of rows in the node schedules
388 * n_band is the current number of completed bands
389 * band_start is the starting row in the node schedules of the current band
390 * root is set if this graph is the original dependence graph,
391 * without any splitting
393 * sorted contains a list of node indices sorted according to the
394 * SCC to which a node belongs
396 * n_edge is the number of edges
397 * edge is the list of edges
398 * max_edge contains the maximal number of edges of each type;
399 * in particular, it contains the number of edges in the inital graph.
400 * edge_table contains pointers into the edge array, hashed on the source
401 * and sink spaces; there is one such table for each type;
402 * a given edge may be referenced from more than one table
403 * if the corresponding relation appears in more than of the
404 * sets of dependences
406 * node_table contains pointers into the node array, hashed on the space
408 * region contains a list of variable sequences that should be non-trivial
410 * lp contains the (I)LP problem used to obtain new schedule rows
412 * src_scc and dst_scc are the source and sink SCCs of an edge with
413 * conflicting constraints
415 * scc represents the number of components
417 struct isl_sched_graph {
418 isl_map_to_basic_set *intra_hmap;
419 isl_map_to_basic_set *inter_hmap;
421 struct isl_sched_node *node;
422 int n;
423 int maxvar;
424 int max_row;
425 int n_row;
427 int *sorted;
429 int n_band;
430 int n_total_row;
431 int band_start;
433 int root;
435 struct isl_sched_edge *edge;
436 int n_edge;
437 int max_edge[isl_edge_last + 1];
438 struct isl_hash_table *edge_table[isl_edge_last + 1];
440 struct isl_hash_table *node_table;
441 struct isl_region *region;
443 isl_basic_set *lp;
445 int src_scc;
446 int dst_scc;
448 int scc;
451 /* Initialize node_table based on the list of nodes.
453 static int graph_init_table(isl_ctx *ctx, struct isl_sched_graph *graph)
455 int i;
457 graph->node_table = isl_hash_table_alloc(ctx, graph->n);
458 if (!graph->node_table)
459 return -1;
461 for (i = 0; i < graph->n; ++i) {
462 struct isl_hash_table_entry *entry;
463 uint32_t hash;
465 hash = isl_space_get_hash(graph->node[i].dim);
466 entry = isl_hash_table_find(ctx, graph->node_table, hash,
467 &node_has_dim,
468 graph->node[i].dim, 1);
469 if (!entry)
470 return -1;
471 entry->data = &graph->node[i];
474 return 0;
477 /* Return a pointer to the node that lives within the given space,
478 * or NULL if there is no such node.
480 static struct isl_sched_node *graph_find_node(isl_ctx *ctx,
481 struct isl_sched_graph *graph, __isl_keep isl_space *dim)
483 struct isl_hash_table_entry *entry;
484 uint32_t hash;
486 hash = isl_space_get_hash(dim);
487 entry = isl_hash_table_find(ctx, graph->node_table, hash,
488 &node_has_dim, dim, 0);
490 return entry ? entry->data : NULL;
493 static int edge_has_src_and_dst(const void *entry, const void *val)
495 const struct isl_sched_edge *edge = entry;
496 const struct isl_sched_edge *temp = val;
498 return edge->src == temp->src && edge->dst == temp->dst;
501 /* Add the given edge to graph->edge_table[type].
503 static int graph_edge_table_add(isl_ctx *ctx, struct isl_sched_graph *graph,
504 enum isl_edge_type type, struct isl_sched_edge *edge)
506 struct isl_hash_table_entry *entry;
507 uint32_t hash;
509 hash = isl_hash_init();
510 hash = isl_hash_builtin(hash, edge->src);
511 hash = isl_hash_builtin(hash, edge->dst);
512 entry = isl_hash_table_find(ctx, graph->edge_table[type], hash,
513 &edge_has_src_and_dst, edge, 1);
514 if (!entry)
515 return -1;
516 entry->data = edge;
518 return 0;
521 /* Allocate the edge_tables based on the maximal number of edges of
522 * each type.
524 static int graph_init_edge_tables(isl_ctx *ctx, struct isl_sched_graph *graph)
526 int i;
528 for (i = 0; i <= isl_edge_last; ++i) {
529 graph->edge_table[i] = isl_hash_table_alloc(ctx,
530 graph->max_edge[i]);
531 if (!graph->edge_table[i])
532 return -1;
535 return 0;
538 /* If graph->edge_table[type] contains an edge from the given source
539 * to the given destination, then return the hash table entry of this edge.
540 * Otherwise, return NULL.
542 static struct isl_hash_table_entry *graph_find_edge_entry(
543 struct isl_sched_graph *graph,
544 enum isl_edge_type type,
545 struct isl_sched_node *src, struct isl_sched_node *dst)
547 isl_ctx *ctx = isl_space_get_ctx(src->dim);
548 uint32_t hash;
549 struct isl_sched_edge temp = { .src = src, .dst = dst };
551 hash = isl_hash_init();
552 hash = isl_hash_builtin(hash, temp.src);
553 hash = isl_hash_builtin(hash, temp.dst);
554 return isl_hash_table_find(ctx, graph->edge_table[type], hash,
555 &edge_has_src_and_dst, &temp, 0);
559 /* If graph->edge_table[type] contains an edge from the given source
560 * to the given destination, then return this edge.
561 * Otherwise, return NULL.
563 static struct isl_sched_edge *graph_find_edge(struct isl_sched_graph *graph,
564 enum isl_edge_type type,
565 struct isl_sched_node *src, struct isl_sched_node *dst)
567 struct isl_hash_table_entry *entry;
569 entry = graph_find_edge_entry(graph, type, src, dst);
570 if (!entry)
571 return NULL;
573 return entry->data;
576 /* Check whether the dependence graph has an edge of the given type
577 * between the given two nodes.
579 static int graph_has_edge(struct isl_sched_graph *graph,
580 enum isl_edge_type type,
581 struct isl_sched_node *src, struct isl_sched_node *dst)
583 struct isl_sched_edge *edge;
584 int empty;
586 edge = graph_find_edge(graph, type, src, dst);
587 if (!edge)
588 return 0;
590 empty = isl_map_plain_is_empty(edge->map);
591 if (empty < 0)
592 return -1;
594 return !empty;
597 /* Look for any edge with the same src, dst and map fields as "model".
599 * Return the matching edge if one can be found.
600 * Return "model" if no matching edge is found.
601 * Return NULL on error.
603 static struct isl_sched_edge *graph_find_matching_edge(
604 struct isl_sched_graph *graph, struct isl_sched_edge *model)
606 enum isl_edge_type i;
607 struct isl_sched_edge *edge;
609 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
610 int is_equal;
612 edge = graph_find_edge(graph, i, model->src, model->dst);
613 if (!edge)
614 continue;
615 is_equal = isl_map_plain_is_equal(model->map, edge->map);
616 if (is_equal < 0)
617 return NULL;
618 if (is_equal)
619 return edge;
622 return model;
625 /* Remove the given edge from all the edge_tables that refer to it.
627 static void graph_remove_edge(struct isl_sched_graph *graph,
628 struct isl_sched_edge *edge)
630 isl_ctx *ctx = isl_map_get_ctx(edge->map);
631 enum isl_edge_type i;
633 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
634 struct isl_hash_table_entry *entry;
636 entry = graph_find_edge_entry(graph, i, edge->src, edge->dst);
637 if (!entry)
638 continue;
639 if (entry->data != edge)
640 continue;
641 isl_hash_table_remove(ctx, graph->edge_table[i], entry);
645 /* Check whether the dependence graph has any edge
646 * between the given two nodes.
648 static int graph_has_any_edge(struct isl_sched_graph *graph,
649 struct isl_sched_node *src, struct isl_sched_node *dst)
651 enum isl_edge_type i;
652 int r;
654 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
655 r = graph_has_edge(graph, i, src, dst);
656 if (r < 0 || r)
657 return r;
660 return r;
663 /* Check whether the dependence graph has a validity edge
664 * between the given two nodes.
666 * Conditional validity edges are essentially validity edges that
667 * can be ignored if the corresponding condition edges are iteration private.
668 * Here, we are only checking for the presence of validity
669 * edges, so we need to consider the conditional validity edges too.
670 * In particular, this function is used during the detection
671 * of strongly connected components and we cannot ignore
672 * conditional validity edges during this detection.
674 static int graph_has_validity_edge(struct isl_sched_graph *graph,
675 struct isl_sched_node *src, struct isl_sched_node *dst)
677 int r;
679 r = graph_has_edge(graph, isl_edge_validity, src, dst);
680 if (r < 0 || r)
681 return r;
683 return graph_has_edge(graph, isl_edge_conditional_validity, src, dst);
686 static int graph_alloc(isl_ctx *ctx, struct isl_sched_graph *graph,
687 int n_node, int n_edge)
689 int i;
691 graph->n = n_node;
692 graph->n_edge = n_edge;
693 graph->node = isl_calloc_array(ctx, struct isl_sched_node, graph->n);
694 graph->sorted = isl_calloc_array(ctx, int, graph->n);
695 graph->region = isl_alloc_array(ctx, struct isl_region, graph->n);
696 graph->edge = isl_calloc_array(ctx,
697 struct isl_sched_edge, graph->n_edge);
699 graph->intra_hmap = isl_map_to_basic_set_alloc(ctx, 2 * n_edge);
700 graph->inter_hmap = isl_map_to_basic_set_alloc(ctx, 2 * n_edge);
702 if (!graph->node || !graph->region || (graph->n_edge && !graph->edge) ||
703 !graph->sorted)
704 return -1;
706 for(i = 0; i < graph->n; ++i)
707 graph->sorted[i] = i;
709 return 0;
712 static void graph_free(isl_ctx *ctx, struct isl_sched_graph *graph)
714 int i;
716 isl_map_to_basic_set_free(graph->intra_hmap);
717 isl_map_to_basic_set_free(graph->inter_hmap);
719 if (graph->node)
720 for (i = 0; i < graph->n; ++i) {
721 isl_space_free(graph->node[i].dim);
722 isl_mat_free(graph->node[i].sched);
723 isl_map_free(graph->node[i].sched_map);
724 isl_mat_free(graph->node[i].cmap);
725 isl_mat_free(graph->node[i].cinv);
726 if (graph->root) {
727 free(graph->node[i].band);
728 free(graph->node[i].band_id);
729 free(graph->node[i].coincident);
732 free(graph->node);
733 free(graph->sorted);
734 if (graph->edge)
735 for (i = 0; i < graph->n_edge; ++i) {
736 isl_map_free(graph->edge[i].map);
737 isl_union_map_free(graph->edge[i].tagged_condition);
738 isl_union_map_free(graph->edge[i].tagged_validity);
740 free(graph->edge);
741 free(graph->region);
742 for (i = 0; i <= isl_edge_last; ++i)
743 isl_hash_table_free(ctx, graph->edge_table[i]);
744 isl_hash_table_free(ctx, graph->node_table);
745 isl_basic_set_free(graph->lp);
748 /* For each "set" on which this function is called, increment
749 * graph->n by one and update graph->maxvar.
751 static int init_n_maxvar(__isl_take isl_set *set, void *user)
753 struct isl_sched_graph *graph = user;
754 int nvar = isl_set_dim(set, isl_dim_set);
756 graph->n++;
757 if (nvar > graph->maxvar)
758 graph->maxvar = nvar;
760 isl_set_free(set);
762 return 0;
765 /* Compute the number of rows that should be allocated for the schedule.
766 * The graph can be split at most "n - 1" times, there can be at most
767 * two rows for each dimension in the iteration domains (in particular,
768 * we usually have one row, but it may be split by split_scaled),
769 * and there can be one extra row for ordering the statements.
770 * Note that if we have actually split "n - 1" times, then no ordering
771 * is needed, so in principle we could use "graph->n + 2 * graph->maxvar - 1".
773 static int compute_max_row(struct isl_sched_graph *graph,
774 __isl_keep isl_union_set *domain)
776 graph->n = 0;
777 graph->maxvar = 0;
778 if (isl_union_set_foreach_set(domain, &init_n_maxvar, graph) < 0)
779 return -1;
780 graph->max_row = graph->n + 2 * graph->maxvar;
782 return 0;
785 /* Add a new node to the graph representing the given set.
787 static int extract_node(__isl_take isl_set *set, void *user)
789 int nvar, nparam;
790 isl_ctx *ctx;
791 isl_space *space;
792 isl_mat *sched;
793 struct isl_sched_graph *graph = user;
794 int *band, *band_id, *coincident;
796 ctx = isl_set_get_ctx(set);
797 space = isl_set_get_space(set);
798 isl_set_free(set);
799 nvar = isl_space_dim(space, isl_dim_set);
800 nparam = isl_space_dim(space, isl_dim_param);
801 if (!ctx->opt->schedule_parametric)
802 nparam = 0;
803 sched = isl_mat_alloc(ctx, 0, 1 + nparam + nvar);
804 graph->node[graph->n].dim = space;
805 graph->node[graph->n].nvar = nvar;
806 graph->node[graph->n].nparam = nparam;
807 graph->node[graph->n].sched = sched;
808 graph->node[graph->n].sched_map = NULL;
809 band = isl_alloc_array(ctx, int, graph->max_row);
810 graph->node[graph->n].band = band;
811 band_id = isl_calloc_array(ctx, int, graph->max_row);
812 graph->node[graph->n].band_id = band_id;
813 coincident = isl_calloc_array(ctx, int, graph->max_row);
814 graph->node[graph->n].coincident = coincident;
815 graph->n++;
817 if (!space || !sched ||
818 (graph->max_row && (!band || !band_id || !coincident)))
819 return -1;
821 return 0;
824 struct isl_extract_edge_data {
825 enum isl_edge_type type;
826 struct isl_sched_graph *graph;
829 /* Merge edge2 into edge1, freeing the contents of edge2.
830 * "type" is the type of the schedule constraint from which edge2 was
831 * extracted.
832 * Return 0 on success and -1 on failure.
834 * edge1 and edge2 are assumed to have the same value for the map field.
836 static int merge_edge(enum isl_edge_type type, struct isl_sched_edge *edge1,
837 struct isl_sched_edge *edge2)
839 edge1->validity |= edge2->validity;
840 edge1->coincidence |= edge2->coincidence;
841 edge1->proximity |= edge2->proximity;
842 edge1->condition |= edge2->condition;
843 edge1->conditional_validity |= edge2->conditional_validity;
844 isl_map_free(edge2->map);
846 if (type == isl_edge_condition) {
847 if (!edge1->tagged_condition)
848 edge1->tagged_condition = edge2->tagged_condition;
849 else
850 edge1->tagged_condition =
851 isl_union_map_union(edge1->tagged_condition,
852 edge2->tagged_condition);
855 if (type == isl_edge_conditional_validity) {
856 if (!edge1->tagged_validity)
857 edge1->tagged_validity = edge2->tagged_validity;
858 else
859 edge1->tagged_validity =
860 isl_union_map_union(edge1->tagged_validity,
861 edge2->tagged_validity);
864 if (type == isl_edge_condition && !edge1->tagged_condition)
865 return -1;
866 if (type == isl_edge_conditional_validity && !edge1->tagged_validity)
867 return -1;
869 return 0;
872 /* Insert dummy tags in domain and range of "map".
874 * In particular, if "map" is of the form
876 * A -> B
878 * then return
880 * [A -> dummy_tag] -> [B -> dummy_tag]
882 * where the dummy_tags are identical and equal to any dummy tags
883 * introduced by any other call to this function.
885 static __isl_give isl_map *insert_dummy_tags(__isl_take isl_map *map)
887 static char dummy;
888 isl_ctx *ctx;
889 isl_id *id;
890 isl_space *space;
891 isl_set *domain, *range;
893 ctx = isl_map_get_ctx(map);
895 id = isl_id_alloc(ctx, NULL, &dummy);
896 space = isl_space_params(isl_map_get_space(map));
897 space = isl_space_set_from_params(space);
898 space = isl_space_set_tuple_id(space, isl_dim_set, id);
899 space = isl_space_map_from_set(space);
901 domain = isl_map_wrap(map);
902 range = isl_map_wrap(isl_map_universe(space));
903 map = isl_map_from_domain_and_range(domain, range);
904 map = isl_map_zip(map);
906 return map;
909 /* Add a new edge to the graph based on the given map
910 * and add it to data->graph->edge_table[data->type].
911 * If a dependence relation of a given type happens to be identical
912 * to one of the dependence relations of a type that was added before,
913 * then we don't create a new edge, but instead mark the original edge
914 * as also representing a dependence of the current type.
916 * Edges of type isl_edge_condition or isl_edge_conditional_validity
917 * may be specified as "tagged" dependence relations. That is, "map"
918 * may contain elements (i -> a) -> (j -> b), where i -> j denotes
919 * the dependence on iterations and a and b are tags.
920 * edge->map is set to the relation containing the elements i -> j,
921 * while edge->tagged_condition and edge->tagged_validity contain
922 * the union of all the "map" relations
923 * for which extract_edge is called that result in the same edge->map.
925 static int extract_edge(__isl_take isl_map *map, void *user)
927 isl_ctx *ctx = isl_map_get_ctx(map);
928 struct isl_extract_edge_data *data = user;
929 struct isl_sched_graph *graph = data->graph;
930 struct isl_sched_node *src, *dst;
931 isl_space *dim;
932 struct isl_sched_edge *edge;
933 isl_map *tagged = NULL;
935 if (data->type == isl_edge_condition ||
936 data->type == isl_edge_conditional_validity) {
937 if (isl_map_can_zip(map)) {
938 tagged = isl_map_copy(map);
939 map = isl_set_unwrap(isl_map_domain(isl_map_zip(map)));
940 } else {
941 tagged = insert_dummy_tags(isl_map_copy(map));
945 dim = isl_space_domain(isl_map_get_space(map));
946 src = graph_find_node(ctx, graph, dim);
947 isl_space_free(dim);
948 dim = isl_space_range(isl_map_get_space(map));
949 dst = graph_find_node(ctx, graph, dim);
950 isl_space_free(dim);
952 if (!src || !dst) {
953 isl_map_free(map);
954 isl_map_free(tagged);
955 return 0;
958 graph->edge[graph->n_edge].src = src;
959 graph->edge[graph->n_edge].dst = dst;
960 graph->edge[graph->n_edge].map = map;
961 graph->edge[graph->n_edge].validity = 0;
962 graph->edge[graph->n_edge].coincidence = 0;
963 graph->edge[graph->n_edge].proximity = 0;
964 graph->edge[graph->n_edge].condition = 0;
965 graph->edge[graph->n_edge].local = 0;
966 graph->edge[graph->n_edge].conditional_validity = 0;
967 graph->edge[graph->n_edge].tagged_condition = NULL;
968 graph->edge[graph->n_edge].tagged_validity = NULL;
969 if (data->type == isl_edge_validity)
970 graph->edge[graph->n_edge].validity = 1;
971 if (data->type == isl_edge_coincidence)
972 graph->edge[graph->n_edge].coincidence = 1;
973 if (data->type == isl_edge_proximity)
974 graph->edge[graph->n_edge].proximity = 1;
975 if (data->type == isl_edge_condition) {
976 graph->edge[graph->n_edge].condition = 1;
977 graph->edge[graph->n_edge].tagged_condition =
978 isl_union_map_from_map(tagged);
980 if (data->type == isl_edge_conditional_validity) {
981 graph->edge[graph->n_edge].conditional_validity = 1;
982 graph->edge[graph->n_edge].tagged_validity =
983 isl_union_map_from_map(tagged);
986 edge = graph_find_matching_edge(graph, &graph->edge[graph->n_edge]);
987 if (!edge) {
988 graph->n_edge++;
989 return -1;
991 if (edge == &graph->edge[graph->n_edge])
992 return graph_edge_table_add(ctx, graph, data->type,
993 &graph->edge[graph->n_edge++]);
995 if (merge_edge(data->type, edge, &graph->edge[graph->n_edge]) < 0)
996 return -1;
998 return graph_edge_table_add(ctx, graph, data->type, edge);
1001 /* Check whether there is any dependence from node[j] to node[i]
1002 * or from node[i] to node[j].
1004 static int node_follows_weak(int i, int j, void *user)
1006 int f;
1007 struct isl_sched_graph *graph = user;
1009 f = graph_has_any_edge(graph, &graph->node[j], &graph->node[i]);
1010 if (f < 0 || f)
1011 return f;
1012 return graph_has_any_edge(graph, &graph->node[i], &graph->node[j]);
1015 /* Check whether there is a (conditional) validity dependence from node[j]
1016 * to node[i], forcing node[i] to follow node[j].
1018 static int node_follows_strong(int i, int j, void *user)
1020 struct isl_sched_graph *graph = user;
1022 return graph_has_validity_edge(graph, &graph->node[j], &graph->node[i]);
1025 /* Use Tarjan's algorithm for computing the strongly connected components
1026 * in the dependence graph (only validity edges).
1027 * If weak is set, we consider the graph to be undirected and
1028 * we effectively compute the (weakly) connected components.
1029 * Additionally, we also consider other edges when weak is set.
1031 static int detect_ccs(isl_ctx *ctx, struct isl_sched_graph *graph, int weak)
1033 int i, n;
1034 struct isl_tarjan_graph *g = NULL;
1036 g = isl_tarjan_graph_init(ctx, graph->n,
1037 weak ? &node_follows_weak : &node_follows_strong, graph);
1038 if (!g)
1039 return -1;
1041 graph->scc = 0;
1042 i = 0;
1043 n = graph->n;
1044 while (n) {
1045 while (g->order[i] != -1) {
1046 graph->node[g->order[i]].scc = graph->scc;
1047 --n;
1048 ++i;
1050 ++i;
1051 graph->scc++;
1054 isl_tarjan_graph_free(g);
1056 return 0;
1059 /* Apply Tarjan's algorithm to detect the strongly connected components
1060 * in the dependence graph.
1062 static int detect_sccs(isl_ctx *ctx, struct isl_sched_graph *graph)
1064 return detect_ccs(ctx, graph, 0);
1067 /* Apply Tarjan's algorithm to detect the (weakly) connected components
1068 * in the dependence graph.
1070 static int detect_wccs(isl_ctx *ctx, struct isl_sched_graph *graph)
1072 return detect_ccs(ctx, graph, 1);
1075 static int cmp_scc(const void *a, const void *b, void *data)
1077 struct isl_sched_graph *graph = data;
1078 const int *i1 = a;
1079 const int *i2 = b;
1081 return graph->node[*i1].scc - graph->node[*i2].scc;
1084 /* Sort the elements of graph->sorted according to the corresponding SCCs.
1086 static int sort_sccs(struct isl_sched_graph *graph)
1088 return isl_sort(graph->sorted, graph->n, sizeof(int), &cmp_scc, graph);
1091 /* Given a dependence relation R from a node to itself,
1092 * construct the set of coefficients of valid constraints for elements
1093 * in that dependence relation.
1094 * In particular, the result contains tuples of coefficients
1095 * c_0, c_n, c_x such that
1097 * c_0 + c_n n + c_x y - c_x x >= 0 for each (x,y) in R
1099 * or, equivalently,
1101 * c_0 + c_n n + c_x d >= 0 for each d in delta R = { y - x | (x,y) in R }
1103 * We choose here to compute the dual of delta R.
1104 * Alternatively, we could have computed the dual of R, resulting
1105 * in a set of tuples c_0, c_n, c_x, c_y, and then
1106 * plugged in (c_0, c_n, c_x, -c_x).
1108 static __isl_give isl_basic_set *intra_coefficients(
1109 struct isl_sched_graph *graph, __isl_take isl_map *map)
1111 isl_set *delta;
1112 isl_basic_set *coef;
1114 if (isl_map_to_basic_set_has(graph->intra_hmap, map))
1115 return isl_map_to_basic_set_get(graph->intra_hmap, map);
1117 delta = isl_set_remove_divs(isl_map_deltas(isl_map_copy(map)));
1118 coef = isl_set_coefficients(delta);
1119 graph->intra_hmap = isl_map_to_basic_set_set(graph->intra_hmap, map,
1120 isl_basic_set_copy(coef));
1122 return coef;
1125 /* Given a dependence relation R, construct the set of coefficients
1126 * of valid constraints for elements in that dependence relation.
1127 * In particular, the result contains tuples of coefficients
1128 * c_0, c_n, c_x, c_y such that
1130 * c_0 + c_n n + c_x x + c_y y >= 0 for each (x,y) in R
1133 static __isl_give isl_basic_set *inter_coefficients(
1134 struct isl_sched_graph *graph, __isl_take isl_map *map)
1136 isl_set *set;
1137 isl_basic_set *coef;
1139 if (isl_map_to_basic_set_has(graph->inter_hmap, map))
1140 return isl_map_to_basic_set_get(graph->inter_hmap, map);
1142 set = isl_map_wrap(isl_map_remove_divs(isl_map_copy(map)));
1143 coef = isl_set_coefficients(set);
1144 graph->inter_hmap = isl_map_to_basic_set_set(graph->inter_hmap, map,
1145 isl_basic_set_copy(coef));
1147 return coef;
1150 /* Add constraints to graph->lp that force validity for the given
1151 * dependence from a node i to itself.
1152 * That is, add constraints that enforce
1154 * (c_i_0 + c_i_n n + c_i_x y) - (c_i_0 + c_i_n n + c_i_x x)
1155 * = c_i_x (y - x) >= 0
1157 * for each (x,y) in R.
1158 * We obtain general constraints on coefficients (c_0, c_n, c_x)
1159 * of valid constraints for (y - x) and then plug in (0, 0, c_i_x^+ - c_i_x^-),
1160 * where c_i_x = c_i_x^+ - c_i_x^-, with c_i_x^+ and c_i_x^- non-negative.
1161 * In graph->lp, the c_i_x^- appear before their c_i_x^+ counterpart.
1163 * Actually, we do not construct constraints for the c_i_x themselves,
1164 * but for the coefficients of c_i_x written as a linear combination
1165 * of the columns in node->cmap.
1167 static int add_intra_validity_constraints(struct isl_sched_graph *graph,
1168 struct isl_sched_edge *edge)
1170 unsigned total;
1171 isl_map *map = isl_map_copy(edge->map);
1172 isl_ctx *ctx = isl_map_get_ctx(map);
1173 isl_space *dim;
1174 isl_dim_map *dim_map;
1175 isl_basic_set *coef;
1176 struct isl_sched_node *node = edge->src;
1178 coef = intra_coefficients(graph, map);
1180 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
1182 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1183 isl_space_dim(dim, isl_dim_set), isl_mat_copy(node->cmap));
1184 if (!coef)
1185 goto error;
1187 total = isl_basic_set_total_dim(graph->lp);
1188 dim_map = isl_dim_map_alloc(ctx, total);
1189 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 1, 2,
1190 isl_space_dim(dim, isl_dim_set), 1,
1191 node->nvar, -1);
1192 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 2, 2,
1193 isl_space_dim(dim, isl_dim_set), 1,
1194 node->nvar, 1);
1195 graph->lp = isl_basic_set_extend_constraints(graph->lp,
1196 coef->n_eq, coef->n_ineq);
1197 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
1198 coef, dim_map);
1199 isl_space_free(dim);
1201 return 0;
1202 error:
1203 isl_space_free(dim);
1204 return -1;
1207 /* Add constraints to graph->lp that force validity for the given
1208 * dependence from node i to node j.
1209 * That is, add constraints that enforce
1211 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) >= 0
1213 * for each (x,y) in R.
1214 * We obtain general constraints on coefficients (c_0, c_n, c_x, c_y)
1215 * of valid constraints for R and then plug in
1216 * (c_j_0 - c_i_0, c_j_n^+ - c_j_n^- - (c_i_n^+ - c_i_n^-),
1217 * c_j_x^+ - c_j_x^- - (c_i_x^+ - c_i_x^-)),
1218 * where c_* = c_*^+ - c_*^-, with c_*^+ and c_*^- non-negative.
1219 * In graph->lp, the c_*^- appear before their c_*^+ counterpart.
1221 * Actually, we do not construct constraints for the c_*_x themselves,
1222 * but for the coefficients of c_*_x written as a linear combination
1223 * of the columns in node->cmap.
1225 static int add_inter_validity_constraints(struct isl_sched_graph *graph,
1226 struct isl_sched_edge *edge)
1228 unsigned total;
1229 isl_map *map = isl_map_copy(edge->map);
1230 isl_ctx *ctx = isl_map_get_ctx(map);
1231 isl_space *dim;
1232 isl_dim_map *dim_map;
1233 isl_basic_set *coef;
1234 struct isl_sched_node *src = edge->src;
1235 struct isl_sched_node *dst = edge->dst;
1237 coef = inter_coefficients(graph, map);
1239 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
1241 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1242 isl_space_dim(dim, isl_dim_set), isl_mat_copy(src->cmap));
1243 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1244 isl_space_dim(dim, isl_dim_set) + src->nvar,
1245 isl_mat_copy(dst->cmap));
1246 if (!coef)
1247 goto error;
1249 total = isl_basic_set_total_dim(graph->lp);
1250 dim_map = isl_dim_map_alloc(ctx, total);
1252 isl_dim_map_range(dim_map, dst->start, 0, 0, 0, 1, 1);
1253 isl_dim_map_range(dim_map, dst->start + 1, 2, 1, 1, dst->nparam, -1);
1254 isl_dim_map_range(dim_map, dst->start + 2, 2, 1, 1, dst->nparam, 1);
1255 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 1, 2,
1256 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
1257 dst->nvar, -1);
1258 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 2, 2,
1259 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
1260 dst->nvar, 1);
1262 isl_dim_map_range(dim_map, src->start, 0, 0, 0, 1, -1);
1263 isl_dim_map_range(dim_map, src->start + 1, 2, 1, 1, src->nparam, 1);
1264 isl_dim_map_range(dim_map, src->start + 2, 2, 1, 1, src->nparam, -1);
1265 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 1, 2,
1266 isl_space_dim(dim, isl_dim_set), 1,
1267 src->nvar, 1);
1268 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 2, 2,
1269 isl_space_dim(dim, isl_dim_set), 1,
1270 src->nvar, -1);
1272 edge->start = graph->lp->n_ineq;
1273 graph->lp = isl_basic_set_extend_constraints(graph->lp,
1274 coef->n_eq, coef->n_ineq);
1275 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
1276 coef, dim_map);
1277 if (!graph->lp)
1278 goto error;
1279 isl_space_free(dim);
1280 edge->end = graph->lp->n_ineq;
1282 return 0;
1283 error:
1284 isl_space_free(dim);
1285 return -1;
1288 /* Add constraints to graph->lp that bound the dependence distance for the given
1289 * dependence from a node i to itself.
1290 * If s = 1, we add the constraint
1292 * c_i_x (y - x) <= m_0 + m_n n
1294 * or
1296 * -c_i_x (y - x) + m_0 + m_n n >= 0
1298 * for each (x,y) in R.
1299 * If s = -1, we add the constraint
1301 * -c_i_x (y - x) <= m_0 + m_n n
1303 * or
1305 * c_i_x (y - x) + m_0 + m_n n >= 0
1307 * for each (x,y) in R.
1308 * We obtain general constraints on coefficients (c_0, c_n, c_x)
1309 * of valid constraints for (y - x) and then plug in (m_0, m_n, -s * c_i_x),
1310 * with each coefficient (except m_0) represented as a pair of non-negative
1311 * coefficients.
1313 * Actually, we do not construct constraints for the c_i_x themselves,
1314 * but for the coefficients of c_i_x written as a linear combination
1315 * of the columns in node->cmap.
1318 * If "local" is set, then we add constraints
1320 * c_i_x (y - x) <= 0
1322 * or
1324 * -c_i_x (y - x) <= 0
1326 * instead, forcing the dependence distance to be (less than or) equal to 0.
1327 * That is, we plug in (0, 0, -s * c_i_x),
1328 * Note that dependences marked local are treated as validity constraints
1329 * by add_all_validity_constraints and therefore also have
1330 * their distances bounded by 0 from below.
1332 static int add_intra_proximity_constraints(struct isl_sched_graph *graph,
1333 struct isl_sched_edge *edge, int s, int local)
1335 unsigned total;
1336 unsigned nparam;
1337 isl_map *map = isl_map_copy(edge->map);
1338 isl_ctx *ctx = isl_map_get_ctx(map);
1339 isl_space *dim;
1340 isl_dim_map *dim_map;
1341 isl_basic_set *coef;
1342 struct isl_sched_node *node = edge->src;
1344 coef = intra_coefficients(graph, map);
1346 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
1348 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1349 isl_space_dim(dim, isl_dim_set), isl_mat_copy(node->cmap));
1350 if (!coef)
1351 goto error;
1353 nparam = isl_space_dim(node->dim, isl_dim_param);
1354 total = isl_basic_set_total_dim(graph->lp);
1355 dim_map = isl_dim_map_alloc(ctx, total);
1357 if (!local) {
1358 isl_dim_map_range(dim_map, 1, 0, 0, 0, 1, 1);
1359 isl_dim_map_range(dim_map, 4, 2, 1, 1, nparam, -1);
1360 isl_dim_map_range(dim_map, 5, 2, 1, 1, nparam, 1);
1362 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 1, 2,
1363 isl_space_dim(dim, isl_dim_set), 1,
1364 node->nvar, s);
1365 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 2, 2,
1366 isl_space_dim(dim, isl_dim_set), 1,
1367 node->nvar, -s);
1368 graph->lp = isl_basic_set_extend_constraints(graph->lp,
1369 coef->n_eq, coef->n_ineq);
1370 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
1371 coef, dim_map);
1372 isl_space_free(dim);
1374 return 0;
1375 error:
1376 isl_space_free(dim);
1377 return -1;
1380 /* Add constraints to graph->lp that bound the dependence distance for the given
1381 * dependence from node i to node j.
1382 * If s = 1, we add the constraint
1384 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x)
1385 * <= m_0 + m_n n
1387 * or
1389 * -(c_j_0 + c_j_n n + c_j_x y) + (c_i_0 + c_i_n n + c_i_x x) +
1390 * m_0 + m_n n >= 0
1392 * for each (x,y) in R.
1393 * If s = -1, we add the constraint
1395 * -((c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x))
1396 * <= m_0 + m_n n
1398 * or
1400 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) +
1401 * m_0 + m_n n >= 0
1403 * for each (x,y) in R.
1404 * We obtain general constraints on coefficients (c_0, c_n, c_x, c_y)
1405 * of valid constraints for R and then plug in
1406 * (m_0 - s*c_j_0 + s*c_i_0, m_n - s*c_j_n + s*c_i_n,
1407 * -s*c_j_x+s*c_i_x)
1408 * with each coefficient (except m_0, c_j_0 and c_i_0)
1409 * represented as a pair of non-negative coefficients.
1411 * Actually, we do not construct constraints for the c_*_x themselves,
1412 * but for the coefficients of c_*_x written as a linear combination
1413 * of the columns in node->cmap.
1416 * If "local" is set, then we add constraints
1418 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) <= 0
1420 * or
1422 * -((c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x)) <= 0
1424 * instead, forcing the dependence distance to be (less than or) equal to 0.
1425 * That is, we plug in
1426 * (-s*c_j_0 + s*c_i_0, -s*c_j_n + s*c_i_n, -s*c_j_x+s*c_i_x).
1427 * Note that dependences marked local are treated as validity constraints
1428 * by add_all_validity_constraints and therefore also have
1429 * their distances bounded by 0 from below.
1431 static int add_inter_proximity_constraints(struct isl_sched_graph *graph,
1432 struct isl_sched_edge *edge, int s, int local)
1434 unsigned total;
1435 unsigned nparam;
1436 isl_map *map = isl_map_copy(edge->map);
1437 isl_ctx *ctx = isl_map_get_ctx(map);
1438 isl_space *dim;
1439 isl_dim_map *dim_map;
1440 isl_basic_set *coef;
1441 struct isl_sched_node *src = edge->src;
1442 struct isl_sched_node *dst = edge->dst;
1444 coef = inter_coefficients(graph, map);
1446 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
1448 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1449 isl_space_dim(dim, isl_dim_set), isl_mat_copy(src->cmap));
1450 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1451 isl_space_dim(dim, isl_dim_set) + src->nvar,
1452 isl_mat_copy(dst->cmap));
1453 if (!coef)
1454 goto error;
1456 nparam = isl_space_dim(src->dim, isl_dim_param);
1457 total = isl_basic_set_total_dim(graph->lp);
1458 dim_map = isl_dim_map_alloc(ctx, total);
1460 if (!local) {
1461 isl_dim_map_range(dim_map, 1, 0, 0, 0, 1, 1);
1462 isl_dim_map_range(dim_map, 4, 2, 1, 1, nparam, -1);
1463 isl_dim_map_range(dim_map, 5, 2, 1, 1, nparam, 1);
1466 isl_dim_map_range(dim_map, dst->start, 0, 0, 0, 1, -s);
1467 isl_dim_map_range(dim_map, dst->start + 1, 2, 1, 1, dst->nparam, s);
1468 isl_dim_map_range(dim_map, dst->start + 2, 2, 1, 1, dst->nparam, -s);
1469 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 1, 2,
1470 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
1471 dst->nvar, s);
1472 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 2, 2,
1473 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
1474 dst->nvar, -s);
1476 isl_dim_map_range(dim_map, src->start, 0, 0, 0, 1, s);
1477 isl_dim_map_range(dim_map, src->start + 1, 2, 1, 1, src->nparam, -s);
1478 isl_dim_map_range(dim_map, src->start + 2, 2, 1, 1, src->nparam, s);
1479 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 1, 2,
1480 isl_space_dim(dim, isl_dim_set), 1,
1481 src->nvar, -s);
1482 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 2, 2,
1483 isl_space_dim(dim, isl_dim_set), 1,
1484 src->nvar, s);
1486 graph->lp = isl_basic_set_extend_constraints(graph->lp,
1487 coef->n_eq, coef->n_ineq);
1488 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
1489 coef, dim_map);
1490 isl_space_free(dim);
1492 return 0;
1493 error:
1494 isl_space_free(dim);
1495 return -1;
1498 /* Add all validity constraints to graph->lp.
1500 * An edge that is forced to be local needs to have its dependence
1501 * distances equal to zero. We take care of bounding them by 0 from below
1502 * here. add_all_proximity_constraints takes care of bounding them by 0
1503 * from above.
1505 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1506 * Otherwise, we ignore them.
1508 static int add_all_validity_constraints(struct isl_sched_graph *graph,
1509 int use_coincidence)
1511 int i;
1513 for (i = 0; i < graph->n_edge; ++i) {
1514 struct isl_sched_edge *edge= &graph->edge[i];
1515 int local;
1517 local = edge->local || (edge->coincidence && use_coincidence);
1518 if (!edge->validity && !local)
1519 continue;
1520 if (edge->src != edge->dst)
1521 continue;
1522 if (add_intra_validity_constraints(graph, edge) < 0)
1523 return -1;
1526 for (i = 0; i < graph->n_edge; ++i) {
1527 struct isl_sched_edge *edge = &graph->edge[i];
1528 int local;
1530 local = edge->local || (edge->coincidence && use_coincidence);
1531 if (!edge->validity && !local)
1532 continue;
1533 if (edge->src == edge->dst)
1534 continue;
1535 if (add_inter_validity_constraints(graph, edge) < 0)
1536 return -1;
1539 return 0;
1542 /* Add constraints to graph->lp that bound the dependence distance
1543 * for all dependence relations.
1544 * If a given proximity dependence is identical to a validity
1545 * dependence, then the dependence distance is already bounded
1546 * from below (by zero), so we only need to bound the distance
1547 * from above. (This includes the case of "local" dependences
1548 * which are treated as validity dependence by add_all_validity_constraints.)
1549 * Otherwise, we need to bound the distance both from above and from below.
1551 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1552 * Otherwise, we ignore them.
1554 static int add_all_proximity_constraints(struct isl_sched_graph *graph,
1555 int use_coincidence)
1557 int i;
1559 for (i = 0; i < graph->n_edge; ++i) {
1560 struct isl_sched_edge *edge= &graph->edge[i];
1561 int local;
1563 local = edge->local || (edge->coincidence && use_coincidence);
1564 if (!edge->proximity && !local)
1565 continue;
1566 if (edge->src == edge->dst &&
1567 add_intra_proximity_constraints(graph, edge, 1, local) < 0)
1568 return -1;
1569 if (edge->src != edge->dst &&
1570 add_inter_proximity_constraints(graph, edge, 1, local) < 0)
1571 return -1;
1572 if (edge->validity || local)
1573 continue;
1574 if (edge->src == edge->dst &&
1575 add_intra_proximity_constraints(graph, edge, -1, 0) < 0)
1576 return -1;
1577 if (edge->src != edge->dst &&
1578 add_inter_proximity_constraints(graph, edge, -1, 0) < 0)
1579 return -1;
1582 return 0;
1585 /* Compute a basis for the rows in the linear part of the schedule
1586 * and extend this basis to a full basis. The remaining rows
1587 * can then be used to force linear independence from the rows
1588 * in the schedule.
1590 * In particular, given the schedule rows S, we compute
1592 * S = H Q
1593 * S U = H
1595 * with H the Hermite normal form of S. That is, all but the
1596 * first rank columns of H are zero and so each row in S is
1597 * a linear combination of the first rank rows of Q.
1598 * The matrix Q is then transposed because we will write the
1599 * coefficients of the next schedule row as a column vector s
1600 * and express this s as a linear combination s = Q c of the
1601 * computed basis.
1602 * Similarly, the matrix U is transposed such that we can
1603 * compute the coefficients c = U s from a schedule row s.
1605 static int node_update_cmap(struct isl_sched_node *node)
1607 isl_mat *H, *U, *Q;
1608 int n_row = isl_mat_rows(node->sched);
1610 H = isl_mat_sub_alloc(node->sched, 0, n_row,
1611 1 + node->nparam, node->nvar);
1613 H = isl_mat_left_hermite(H, 0, &U, &Q);
1614 isl_mat_free(node->cmap);
1615 isl_mat_free(node->cinv);
1616 node->cmap = isl_mat_transpose(Q);
1617 node->cinv = isl_mat_transpose(U);
1618 node->rank = isl_mat_initial_non_zero_cols(H);
1619 isl_mat_free(H);
1621 if (!node->cmap || !node->cinv || node->rank < 0)
1622 return -1;
1623 return 0;
1626 /* How many times should we count the constraints in "edge"?
1628 * If carry is set, then we are counting the number of
1629 * (validity or conditional validity) constraints that will be added
1630 * in setup_carry_lp and we count each edge exactly once.
1632 * Otherwise, we count as follows
1633 * validity -> 1 (>= 0)
1634 * validity+proximity -> 2 (>= 0 and upper bound)
1635 * proximity -> 2 (lower and upper bound)
1636 * local(+any) -> 2 (>= 0 and <= 0)
1638 * If an edge is only marked conditional_validity then it counts
1639 * as zero since it is only checked afterwards.
1641 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1642 * Otherwise, we ignore them.
1644 static int edge_multiplicity(struct isl_sched_edge *edge, int carry,
1645 int use_coincidence)
1647 if (carry && !edge->validity && !edge->conditional_validity)
1648 return 0;
1649 if (carry)
1650 return 1;
1651 if (edge->proximity || edge->local)
1652 return 2;
1653 if (use_coincidence && edge->coincidence)
1654 return 2;
1655 if (edge->validity)
1656 return 1;
1657 return 0;
1660 /* Count the number of equality and inequality constraints
1661 * that will be added for the given map.
1663 * "use_coincidence" is set if we should take into account coincidence edges.
1665 static int count_map_constraints(struct isl_sched_graph *graph,
1666 struct isl_sched_edge *edge, __isl_take isl_map *map,
1667 int *n_eq, int *n_ineq, int carry, int use_coincidence)
1669 isl_basic_set *coef;
1670 int f = edge_multiplicity(edge, carry, use_coincidence);
1672 if (f == 0) {
1673 isl_map_free(map);
1674 return 0;
1677 if (edge->src == edge->dst)
1678 coef = intra_coefficients(graph, map);
1679 else
1680 coef = inter_coefficients(graph, map);
1681 if (!coef)
1682 return -1;
1683 *n_eq += f * coef->n_eq;
1684 *n_ineq += f * coef->n_ineq;
1685 isl_basic_set_free(coef);
1687 return 0;
1690 /* Count the number of equality and inequality constraints
1691 * that will be added to the main lp problem.
1692 * We count as follows
1693 * validity -> 1 (>= 0)
1694 * validity+proximity -> 2 (>= 0 and upper bound)
1695 * proximity -> 2 (lower and upper bound)
1696 * local(+any) -> 2 (>= 0 and <= 0)
1698 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1699 * Otherwise, we ignore them.
1701 static int count_constraints(struct isl_sched_graph *graph,
1702 int *n_eq, int *n_ineq, int use_coincidence)
1704 int i;
1706 *n_eq = *n_ineq = 0;
1707 for (i = 0; i < graph->n_edge; ++i) {
1708 struct isl_sched_edge *edge= &graph->edge[i];
1709 isl_map *map = isl_map_copy(edge->map);
1711 if (count_map_constraints(graph, edge, map, n_eq, n_ineq,
1712 0, use_coincidence) < 0)
1713 return -1;
1716 return 0;
1719 /* Count the number of constraints that will be added by
1720 * add_bound_coefficient_constraints and increment *n_eq and *n_ineq
1721 * accordingly.
1723 * In practice, add_bound_coefficient_constraints only adds inequalities.
1725 static int count_bound_coefficient_constraints(isl_ctx *ctx,
1726 struct isl_sched_graph *graph, int *n_eq, int *n_ineq)
1728 int i;
1730 if (ctx->opt->schedule_max_coefficient == -1)
1731 return 0;
1733 for (i = 0; i < graph->n; ++i)
1734 *n_ineq += 2 * graph->node[i].nparam + 2 * graph->node[i].nvar;
1736 return 0;
1739 /* Add constraints that bound the values of the variable and parameter
1740 * coefficients of the schedule.
1742 * The maximal value of the coefficients is defined by the option
1743 * 'schedule_max_coefficient'.
1745 static int add_bound_coefficient_constraints(isl_ctx *ctx,
1746 struct isl_sched_graph *graph)
1748 int i, j, k;
1749 int max_coefficient;
1750 int total;
1752 max_coefficient = ctx->opt->schedule_max_coefficient;
1754 if (max_coefficient == -1)
1755 return 0;
1757 total = isl_basic_set_total_dim(graph->lp);
1759 for (i = 0; i < graph->n; ++i) {
1760 struct isl_sched_node *node = &graph->node[i];
1761 for (j = 0; j < 2 * node->nparam + 2 * node->nvar; ++j) {
1762 int dim;
1763 k = isl_basic_set_alloc_inequality(graph->lp);
1764 if (k < 0)
1765 return -1;
1766 dim = 1 + node->start + 1 + j;
1767 isl_seq_clr(graph->lp->ineq[k], 1 + total);
1768 isl_int_set_si(graph->lp->ineq[k][dim], -1);
1769 isl_int_set_si(graph->lp->ineq[k][0], max_coefficient);
1773 return 0;
1776 /* Construct an ILP problem for finding schedule coefficients
1777 * that result in non-negative, but small dependence distances
1778 * over all dependences.
1779 * In particular, the dependence distances over proximity edges
1780 * are bounded by m_0 + m_n n and we compute schedule coefficients
1781 * with small values (preferably zero) of m_n and m_0.
1783 * All variables of the ILP are non-negative. The actual coefficients
1784 * may be negative, so each coefficient is represented as the difference
1785 * of two non-negative variables. The negative part always appears
1786 * immediately before the positive part.
1787 * Other than that, the variables have the following order
1789 * - sum of positive and negative parts of m_n coefficients
1790 * - m_0
1791 * - sum of positive and negative parts of all c_n coefficients
1792 * (unconstrained when computing non-parametric schedules)
1793 * - sum of positive and negative parts of all c_x coefficients
1794 * - positive and negative parts of m_n coefficients
1795 * - for each node
1796 * - c_i_0
1797 * - positive and negative parts of c_i_n (if parametric)
1798 * - positive and negative parts of c_i_x
1800 * The c_i_x are not represented directly, but through the columns of
1801 * node->cmap. That is, the computed values are for variable t_i_x
1802 * such that c_i_x = Q t_i_x with Q equal to node->cmap.
1804 * The constraints are those from the edges plus two or three equalities
1805 * to express the sums.
1807 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1808 * Otherwise, we ignore them.
1810 static int setup_lp(isl_ctx *ctx, struct isl_sched_graph *graph,
1811 int use_coincidence)
1813 int i, j;
1814 int k;
1815 unsigned nparam;
1816 unsigned total;
1817 isl_space *dim;
1818 int parametric;
1819 int param_pos;
1820 int n_eq, n_ineq;
1821 int max_constant_term;
1823 max_constant_term = ctx->opt->schedule_max_constant_term;
1825 parametric = ctx->opt->schedule_parametric;
1826 nparam = isl_space_dim(graph->node[0].dim, isl_dim_param);
1827 param_pos = 4;
1828 total = param_pos + 2 * nparam;
1829 for (i = 0; i < graph->n; ++i) {
1830 struct isl_sched_node *node = &graph->node[graph->sorted[i]];
1831 if (node_update_cmap(node) < 0)
1832 return -1;
1833 node->start = total;
1834 total += 1 + 2 * (node->nparam + node->nvar);
1837 if (count_constraints(graph, &n_eq, &n_ineq, use_coincidence) < 0)
1838 return -1;
1839 if (count_bound_coefficient_constraints(ctx, graph, &n_eq, &n_ineq) < 0)
1840 return -1;
1842 dim = isl_space_set_alloc(ctx, 0, total);
1843 isl_basic_set_free(graph->lp);
1844 n_eq += 2 + parametric;
1845 if (max_constant_term != -1)
1846 n_ineq += graph->n;
1848 graph->lp = isl_basic_set_alloc_space(dim, 0, n_eq, n_ineq);
1850 k = isl_basic_set_alloc_equality(graph->lp);
1851 if (k < 0)
1852 return -1;
1853 isl_seq_clr(graph->lp->eq[k], 1 + total);
1854 isl_int_set_si(graph->lp->eq[k][1], -1);
1855 for (i = 0; i < 2 * nparam; ++i)
1856 isl_int_set_si(graph->lp->eq[k][1 + param_pos + i], 1);
1858 if (parametric) {
1859 k = isl_basic_set_alloc_equality(graph->lp);
1860 if (k < 0)
1861 return -1;
1862 isl_seq_clr(graph->lp->eq[k], 1 + total);
1863 isl_int_set_si(graph->lp->eq[k][3], -1);
1864 for (i = 0; i < graph->n; ++i) {
1865 int pos = 1 + graph->node[i].start + 1;
1867 for (j = 0; j < 2 * graph->node[i].nparam; ++j)
1868 isl_int_set_si(graph->lp->eq[k][pos + j], 1);
1872 k = isl_basic_set_alloc_equality(graph->lp);
1873 if (k < 0)
1874 return -1;
1875 isl_seq_clr(graph->lp->eq[k], 1 + total);
1876 isl_int_set_si(graph->lp->eq[k][4], -1);
1877 for (i = 0; i < graph->n; ++i) {
1878 struct isl_sched_node *node = &graph->node[i];
1879 int pos = 1 + node->start + 1 + 2 * node->nparam;
1881 for (j = 0; j < 2 * node->nvar; ++j)
1882 isl_int_set_si(graph->lp->eq[k][pos + j], 1);
1885 if (max_constant_term != -1)
1886 for (i = 0; i < graph->n; ++i) {
1887 struct isl_sched_node *node = &graph->node[i];
1888 k = isl_basic_set_alloc_inequality(graph->lp);
1889 if (k < 0)
1890 return -1;
1891 isl_seq_clr(graph->lp->ineq[k], 1 + total);
1892 isl_int_set_si(graph->lp->ineq[k][1 + node->start], -1);
1893 isl_int_set_si(graph->lp->ineq[k][0], max_constant_term);
1896 if (add_bound_coefficient_constraints(ctx, graph) < 0)
1897 return -1;
1898 if (add_all_validity_constraints(graph, use_coincidence) < 0)
1899 return -1;
1900 if (add_all_proximity_constraints(graph, use_coincidence) < 0)
1901 return -1;
1903 return 0;
1906 /* Analyze the conflicting constraint found by
1907 * isl_tab_basic_set_non_trivial_lexmin. If it corresponds to the validity
1908 * constraint of one of the edges between distinct nodes, living, moreover
1909 * in distinct SCCs, then record the source and sink SCC as this may
1910 * be a good place to cut between SCCs.
1912 static int check_conflict(int con, void *user)
1914 int i;
1915 struct isl_sched_graph *graph = user;
1917 if (graph->src_scc >= 0)
1918 return 0;
1920 con -= graph->lp->n_eq;
1922 if (con >= graph->lp->n_ineq)
1923 return 0;
1925 for (i = 0; i < graph->n_edge; ++i) {
1926 if (!graph->edge[i].validity)
1927 continue;
1928 if (graph->edge[i].src == graph->edge[i].dst)
1929 continue;
1930 if (graph->edge[i].src->scc == graph->edge[i].dst->scc)
1931 continue;
1932 if (graph->edge[i].start > con)
1933 continue;
1934 if (graph->edge[i].end <= con)
1935 continue;
1936 graph->src_scc = graph->edge[i].src->scc;
1937 graph->dst_scc = graph->edge[i].dst->scc;
1940 return 0;
1943 /* Check whether the next schedule row of the given node needs to be
1944 * non-trivial. Lower-dimensional domains may have some trivial rows,
1945 * but as soon as the number of remaining required non-trivial rows
1946 * is as large as the number or remaining rows to be computed,
1947 * all remaining rows need to be non-trivial.
1949 static int needs_row(struct isl_sched_graph *graph, struct isl_sched_node *node)
1951 return node->nvar - node->rank >= graph->maxvar - graph->n_row;
1954 /* Solve the ILP problem constructed in setup_lp.
1955 * For each node such that all the remaining rows of its schedule
1956 * need to be non-trivial, we construct a non-triviality region.
1957 * This region imposes that the next row is independent of previous rows.
1958 * In particular the coefficients c_i_x are represented by t_i_x
1959 * variables with c_i_x = Q t_i_x and Q a unimodular matrix such that
1960 * its first columns span the rows of the previously computed part
1961 * of the schedule. The non-triviality region enforces that at least
1962 * one of the remaining components of t_i_x is non-zero, i.e.,
1963 * that the new schedule row depends on at least one of the remaining
1964 * columns of Q.
1966 static __isl_give isl_vec *solve_lp(struct isl_sched_graph *graph)
1968 int i;
1969 isl_vec *sol;
1970 isl_basic_set *lp;
1972 for (i = 0; i < graph->n; ++i) {
1973 struct isl_sched_node *node = &graph->node[i];
1974 int skip = node->rank;
1975 graph->region[i].pos = node->start + 1 + 2*(node->nparam+skip);
1976 if (needs_row(graph, node))
1977 graph->region[i].len = 2 * (node->nvar - skip);
1978 else
1979 graph->region[i].len = 0;
1981 lp = isl_basic_set_copy(graph->lp);
1982 sol = isl_tab_basic_set_non_trivial_lexmin(lp, 2, graph->n,
1983 graph->region, &check_conflict, graph);
1984 return sol;
1987 /* Update the schedules of all nodes based on the given solution
1988 * of the LP problem.
1989 * The new row is added to the current band.
1990 * All possibly negative coefficients are encoded as a difference
1991 * of two non-negative variables, so we need to perform the subtraction
1992 * here. Moreover, if use_cmap is set, then the solution does
1993 * not refer to the actual coefficients c_i_x, but instead to variables
1994 * t_i_x such that c_i_x = Q t_i_x and Q is equal to node->cmap.
1995 * In this case, we then also need to perform this multiplication
1996 * to obtain the values of c_i_x.
1998 * If coincident is set, then the caller guarantees that the new
1999 * row satisfies the coincidence constraints.
2001 static int update_schedule(struct isl_sched_graph *graph,
2002 __isl_take isl_vec *sol, int use_cmap, int coincident)
2004 int i, j;
2005 isl_vec *csol = NULL;
2007 if (!sol)
2008 goto error;
2009 if (sol->size == 0)
2010 isl_die(sol->ctx, isl_error_internal,
2011 "no solution found", goto error);
2012 if (graph->n_total_row >= graph->max_row)
2013 isl_die(sol->ctx, isl_error_internal,
2014 "too many schedule rows", goto error);
2016 for (i = 0; i < graph->n; ++i) {
2017 struct isl_sched_node *node = &graph->node[i];
2018 int pos = node->start;
2019 int row = isl_mat_rows(node->sched);
2021 isl_vec_free(csol);
2022 csol = isl_vec_alloc(sol->ctx, node->nvar);
2023 if (!csol)
2024 goto error;
2026 isl_map_free(node->sched_map);
2027 node->sched_map = NULL;
2028 node->sched = isl_mat_add_rows(node->sched, 1);
2029 if (!node->sched)
2030 goto error;
2031 node->sched = isl_mat_set_element(node->sched, row, 0,
2032 sol->el[1 + pos]);
2033 for (j = 0; j < node->nparam + node->nvar; ++j)
2034 isl_int_sub(sol->el[1 + pos + 1 + 2 * j + 1],
2035 sol->el[1 + pos + 1 + 2 * j + 1],
2036 sol->el[1 + pos + 1 + 2 * j]);
2037 for (j = 0; j < node->nparam; ++j)
2038 node->sched = isl_mat_set_element(node->sched,
2039 row, 1 + j, sol->el[1+pos+1+2*j+1]);
2040 for (j = 0; j < node->nvar; ++j)
2041 isl_int_set(csol->el[j],
2042 sol->el[1+pos+1+2*(node->nparam+j)+1]);
2043 if (use_cmap)
2044 csol = isl_mat_vec_product(isl_mat_copy(node->cmap),
2045 csol);
2046 if (!csol)
2047 goto error;
2048 for (j = 0; j < node->nvar; ++j)
2049 node->sched = isl_mat_set_element(node->sched,
2050 row, 1 + node->nparam + j, csol->el[j]);
2051 node->band[graph->n_total_row] = graph->n_band;
2052 node->coincident[graph->n_total_row] = coincident;
2054 isl_vec_free(sol);
2055 isl_vec_free(csol);
2057 graph->n_row++;
2058 graph->n_total_row++;
2060 return 0;
2061 error:
2062 isl_vec_free(sol);
2063 isl_vec_free(csol);
2064 return -1;
2067 /* Convert row "row" of node->sched into an isl_aff living in "ls"
2068 * and return this isl_aff.
2070 static __isl_give isl_aff *extract_schedule_row(__isl_take isl_local_space *ls,
2071 struct isl_sched_node *node, int row)
2073 int j;
2074 isl_int v;
2075 isl_aff *aff;
2077 isl_int_init(v);
2079 aff = isl_aff_zero_on_domain(ls);
2080 isl_mat_get_element(node->sched, row, 0, &v);
2081 aff = isl_aff_set_constant(aff, v);
2082 for (j = 0; j < node->nparam; ++j) {
2083 isl_mat_get_element(node->sched, row, 1 + j, &v);
2084 aff = isl_aff_set_coefficient(aff, isl_dim_param, j, v);
2086 for (j = 0; j < node->nvar; ++j) {
2087 isl_mat_get_element(node->sched, row, 1 + node->nparam + j, &v);
2088 aff = isl_aff_set_coefficient(aff, isl_dim_in, j, v);
2091 isl_int_clear(v);
2093 return aff;
2096 /* Convert node->sched into a multi_aff and return this multi_aff.
2098 static __isl_give isl_multi_aff *node_extract_schedule_multi_aff(
2099 struct isl_sched_node *node)
2101 int i;
2102 isl_space *space;
2103 isl_local_space *ls;
2104 isl_aff *aff;
2105 isl_multi_aff *ma;
2106 int nrow, ncol;
2108 nrow = isl_mat_rows(node->sched);
2109 ncol = isl_mat_cols(node->sched) - 1;
2110 space = isl_space_from_domain(isl_space_copy(node->dim));
2111 space = isl_space_add_dims(space, isl_dim_out, nrow);
2112 ma = isl_multi_aff_zero(space);
2113 ls = isl_local_space_from_space(isl_space_copy(node->dim));
2115 for (i = 0; i < nrow; ++i) {
2116 aff = extract_schedule_row(isl_local_space_copy(ls), node, i);
2117 ma = isl_multi_aff_set_aff(ma, i, aff);
2120 isl_local_space_free(ls);
2122 return ma;
2125 /* Convert node->sched into a map and return this map.
2127 * The result is cached in node->sched_map, which needs to be released
2128 * whenever node->sched is updated.
2130 static __isl_give isl_map *node_extract_schedule(struct isl_sched_node *node)
2132 if (!node->sched_map) {
2133 isl_multi_aff *ma;
2135 ma = node_extract_schedule_multi_aff(node);
2136 node->sched_map = isl_map_from_multi_aff(ma);
2139 return isl_map_copy(node->sched_map);
2142 /* Construct a map that can be used to update a dependence relation
2143 * based on the current schedule.
2144 * That is, construct a map expressing that source and sink
2145 * are executed within the same iteration of the current schedule.
2146 * This map can then be intersected with the dependence relation.
2147 * This is not the most efficient way, but this shouldn't be a critical
2148 * operation.
2150 static __isl_give isl_map *specializer(struct isl_sched_node *src,
2151 struct isl_sched_node *dst)
2153 isl_map *src_sched, *dst_sched;
2155 src_sched = node_extract_schedule(src);
2156 dst_sched = node_extract_schedule(dst);
2157 return isl_map_apply_range(src_sched, isl_map_reverse(dst_sched));
2160 /* Intersect the domains of the nested relations in domain and range
2161 * of "umap" with "map".
2163 static __isl_give isl_union_map *intersect_domains(
2164 __isl_take isl_union_map *umap, __isl_keep isl_map *map)
2166 isl_union_set *uset;
2168 umap = isl_union_map_zip(umap);
2169 uset = isl_union_set_from_set(isl_map_wrap(isl_map_copy(map)));
2170 umap = isl_union_map_intersect_domain(umap, uset);
2171 umap = isl_union_map_zip(umap);
2172 return umap;
2175 /* Update the dependence relation of the given edge based
2176 * on the current schedule.
2177 * If the dependence is carried completely by the current schedule, then
2178 * it is removed from the edge_tables. It is kept in the list of edges
2179 * as otherwise all edge_tables would have to be recomputed.
2181 static int update_edge(struct isl_sched_graph *graph,
2182 struct isl_sched_edge *edge)
2184 isl_map *id;
2186 id = specializer(edge->src, edge->dst);
2187 edge->map = isl_map_intersect(edge->map, isl_map_copy(id));
2188 if (!edge->map)
2189 goto error;
2191 if (edge->tagged_condition) {
2192 edge->tagged_condition =
2193 intersect_domains(edge->tagged_condition, id);
2194 if (!edge->tagged_condition)
2195 goto error;
2197 if (edge->tagged_validity) {
2198 edge->tagged_validity =
2199 intersect_domains(edge->tagged_validity, id);
2200 if (!edge->tagged_validity)
2201 goto error;
2204 isl_map_free(id);
2205 if (isl_map_plain_is_empty(edge->map))
2206 graph_remove_edge(graph, edge);
2208 return 0;
2209 error:
2210 isl_map_free(id);
2211 return -1;
2214 /* Update the dependence relations of all edges based on the current schedule.
2216 static int update_edges(isl_ctx *ctx, struct isl_sched_graph *graph)
2218 int i;
2220 for (i = graph->n_edge - 1; i >= 0; --i) {
2221 if (update_edge(graph, &graph->edge[i]) < 0)
2222 return -1;
2225 return 0;
2228 static void next_band(struct isl_sched_graph *graph)
2230 graph->band_start = graph->n_total_row;
2231 graph->n_band++;
2234 /* Topologically sort statements mapped to the same schedule iteration
2235 * and add a row to the schedule corresponding to this order.
2237 static int sort_statements(isl_ctx *ctx, struct isl_sched_graph *graph)
2239 int i, j;
2241 if (graph->n <= 1)
2242 return 0;
2244 if (update_edges(ctx, graph) < 0)
2245 return -1;
2247 if (graph->n_edge == 0)
2248 return 0;
2250 if (detect_sccs(ctx, graph) < 0)
2251 return -1;
2253 if (graph->n_total_row >= graph->max_row)
2254 isl_die(ctx, isl_error_internal,
2255 "too many schedule rows", return -1);
2257 for (i = 0; i < graph->n; ++i) {
2258 struct isl_sched_node *node = &graph->node[i];
2259 int row = isl_mat_rows(node->sched);
2260 int cols = isl_mat_cols(node->sched);
2262 isl_map_free(node->sched_map);
2263 node->sched_map = NULL;
2264 node->sched = isl_mat_add_rows(node->sched, 1);
2265 if (!node->sched)
2266 return -1;
2267 node->sched = isl_mat_set_element_si(node->sched, row, 0,
2268 node->scc);
2269 for (j = 1; j < cols; ++j)
2270 node->sched = isl_mat_set_element_si(node->sched,
2271 row, j, 0);
2272 node->band[graph->n_total_row] = graph->n_band;
2275 graph->n_total_row++;
2276 next_band(graph);
2278 return 0;
2281 /* Construct an isl_schedule based on the computed schedule stored
2282 * in graph and with parameters specified by dim.
2284 static __isl_give isl_schedule *extract_schedule(struct isl_sched_graph *graph,
2285 __isl_take isl_space *dim)
2287 int i;
2288 isl_ctx *ctx;
2289 isl_schedule *sched = NULL;
2291 if (!dim)
2292 return NULL;
2294 ctx = isl_space_get_ctx(dim);
2295 sched = isl_calloc(ctx, struct isl_schedule,
2296 sizeof(struct isl_schedule) +
2297 (graph->n - 1) * sizeof(struct isl_schedule_node));
2298 if (!sched)
2299 goto error;
2301 sched->ref = 1;
2302 sched->n = graph->n;
2303 sched->n_band = graph->n_band;
2304 sched->n_total_row = graph->n_total_row;
2306 for (i = 0; i < sched->n; ++i) {
2307 int r, b;
2308 int *band_end, *band_id, *coincident;
2310 sched->node[i].sched =
2311 node_extract_schedule_multi_aff(&graph->node[i]);
2312 if (!sched->node[i].sched)
2313 goto error;
2315 sched->node[i].n_band = graph->n_band;
2316 if (graph->n_band == 0)
2317 continue;
2319 band_end = isl_alloc_array(ctx, int, graph->n_band);
2320 band_id = isl_alloc_array(ctx, int, graph->n_band);
2321 coincident = isl_alloc_array(ctx, int, graph->n_total_row);
2322 sched->node[i].band_end = band_end;
2323 sched->node[i].band_id = band_id;
2324 sched->node[i].coincident = coincident;
2325 if (!band_end || !band_id || !coincident)
2326 goto error;
2328 for (r = 0; r < graph->n_total_row; ++r)
2329 coincident[r] = graph->node[i].coincident[r];
2330 for (r = b = 0; r < graph->n_total_row; ++r) {
2331 if (graph->node[i].band[r] == b)
2332 continue;
2333 band_end[b++] = r;
2334 if (graph->node[i].band[r] == -1)
2335 break;
2337 if (r == graph->n_total_row)
2338 band_end[b++] = r;
2339 sched->node[i].n_band = b;
2340 for (--b; b >= 0; --b)
2341 band_id[b] = graph->node[i].band_id[b];
2344 sched->dim = dim;
2346 return sched;
2347 error:
2348 isl_space_free(dim);
2349 isl_schedule_free(sched);
2350 return NULL;
2353 /* Copy nodes that satisfy node_pred from the src dependence graph
2354 * to the dst dependence graph.
2356 static int copy_nodes(struct isl_sched_graph *dst, struct isl_sched_graph *src,
2357 int (*node_pred)(struct isl_sched_node *node, int data), int data)
2359 int i;
2361 dst->n = 0;
2362 for (i = 0; i < src->n; ++i) {
2363 int j;
2365 if (!node_pred(&src->node[i], data))
2366 continue;
2368 j = dst->n;
2369 dst->node[j].dim = isl_space_copy(src->node[i].dim);
2370 dst->node[j].nvar = src->node[i].nvar;
2371 dst->node[j].nparam = src->node[i].nparam;
2372 dst->node[j].sched = isl_mat_copy(src->node[i].sched);
2373 dst->node[j].sched_map = isl_map_copy(src->node[i].sched_map);
2374 dst->node[j].band = src->node[i].band;
2375 dst->node[j].band_id = src->node[i].band_id;
2376 dst->node[j].coincident = src->node[i].coincident;
2377 dst->n++;
2379 if (!dst->node[j].dim || !dst->node[j].sched)
2380 return -1;
2383 return 0;
2386 /* Copy non-empty edges that satisfy edge_pred from the src dependence graph
2387 * to the dst dependence graph.
2388 * If the source or destination node of the edge is not in the destination
2389 * graph, then it must be a backward proximity edge and it should simply
2390 * be ignored.
2392 static int copy_edges(isl_ctx *ctx, struct isl_sched_graph *dst,
2393 struct isl_sched_graph *src,
2394 int (*edge_pred)(struct isl_sched_edge *edge, int data), int data)
2396 int i;
2397 enum isl_edge_type t;
2399 dst->n_edge = 0;
2400 for (i = 0; i < src->n_edge; ++i) {
2401 struct isl_sched_edge *edge = &src->edge[i];
2402 isl_map *map;
2403 isl_union_map *tagged_condition;
2404 isl_union_map *tagged_validity;
2405 struct isl_sched_node *dst_src, *dst_dst;
2407 if (!edge_pred(edge, data))
2408 continue;
2410 if (isl_map_plain_is_empty(edge->map))
2411 continue;
2413 dst_src = graph_find_node(ctx, dst, edge->src->dim);
2414 dst_dst = graph_find_node(ctx, dst, edge->dst->dim);
2415 if (!dst_src || !dst_dst) {
2416 if (edge->validity || edge->conditional_validity)
2417 isl_die(ctx, isl_error_internal,
2418 "backward (conditional) validity edge",
2419 return -1);
2420 continue;
2423 map = isl_map_copy(edge->map);
2424 tagged_condition = isl_union_map_copy(edge->tagged_condition);
2425 tagged_validity = isl_union_map_copy(edge->tagged_validity);
2427 dst->edge[dst->n_edge].src = dst_src;
2428 dst->edge[dst->n_edge].dst = dst_dst;
2429 dst->edge[dst->n_edge].map = map;
2430 dst->edge[dst->n_edge].tagged_condition = tagged_condition;
2431 dst->edge[dst->n_edge].tagged_validity = tagged_validity;
2432 dst->edge[dst->n_edge].validity = edge->validity;
2433 dst->edge[dst->n_edge].proximity = edge->proximity;
2434 dst->edge[dst->n_edge].coincidence = edge->coincidence;
2435 dst->edge[dst->n_edge].condition = edge->condition;
2436 dst->edge[dst->n_edge].conditional_validity =
2437 edge->conditional_validity;
2438 dst->n_edge++;
2440 if (edge->tagged_condition && !tagged_condition)
2441 return -1;
2442 if (edge->tagged_validity && !tagged_validity)
2443 return -1;
2445 for (t = isl_edge_first; t <= isl_edge_last; ++t) {
2446 if (edge !=
2447 graph_find_edge(src, t, edge->src, edge->dst))
2448 continue;
2449 if (graph_edge_table_add(ctx, dst, t,
2450 &dst->edge[dst->n_edge - 1]) < 0)
2451 return -1;
2455 return 0;
2458 /* Given a "src" dependence graph that contains the nodes from "dst"
2459 * that satisfy node_pred, copy the schedule computed in "src"
2460 * for those nodes back to "dst".
2462 static int copy_schedule(struct isl_sched_graph *dst,
2463 struct isl_sched_graph *src,
2464 int (*node_pred)(struct isl_sched_node *node, int data), int data)
2466 int i;
2468 src->n = 0;
2469 for (i = 0; i < dst->n; ++i) {
2470 if (!node_pred(&dst->node[i], data))
2471 continue;
2472 isl_mat_free(dst->node[i].sched);
2473 isl_map_free(dst->node[i].sched_map);
2474 dst->node[i].sched = isl_mat_copy(src->node[src->n].sched);
2475 dst->node[i].sched_map =
2476 isl_map_copy(src->node[src->n].sched_map);
2477 src->n++;
2480 dst->max_row = src->max_row;
2481 dst->n_total_row = src->n_total_row;
2482 dst->n_band = src->n_band;
2484 return 0;
2487 /* Compute the maximal number of variables over all nodes.
2488 * This is the maximal number of linearly independent schedule
2489 * rows that we need to compute.
2490 * Just in case we end up in a part of the dependence graph
2491 * with only lower-dimensional domains, we make sure we will
2492 * compute the required amount of extra linearly independent rows.
2494 static int compute_maxvar(struct isl_sched_graph *graph)
2496 int i;
2498 graph->maxvar = 0;
2499 for (i = 0; i < graph->n; ++i) {
2500 struct isl_sched_node *node = &graph->node[i];
2501 int nvar;
2503 if (node_update_cmap(node) < 0)
2504 return -1;
2505 nvar = node->nvar + graph->n_row - node->rank;
2506 if (nvar > graph->maxvar)
2507 graph->maxvar = nvar;
2510 return 0;
2513 static int compute_schedule(isl_ctx *ctx, struct isl_sched_graph *graph);
2514 static int compute_schedule_wcc(isl_ctx *ctx, struct isl_sched_graph *graph);
2516 /* Compute a schedule for a subgraph of "graph". In particular, for
2517 * the graph composed of nodes that satisfy node_pred and edges that
2518 * that satisfy edge_pred. The caller should precompute the number
2519 * of nodes and edges that satisfy these predicates and pass them along
2520 * as "n" and "n_edge".
2521 * If the subgraph is known to consist of a single component, then wcc should
2522 * be set and then we call compute_schedule_wcc on the constructed subgraph.
2523 * Otherwise, we call compute_schedule, which will check whether the subgraph
2524 * is connected.
2526 static int compute_sub_schedule(isl_ctx *ctx,
2527 struct isl_sched_graph *graph, int n, int n_edge,
2528 int (*node_pred)(struct isl_sched_node *node, int data),
2529 int (*edge_pred)(struct isl_sched_edge *edge, int data),
2530 int data, int wcc)
2532 struct isl_sched_graph split = { 0 };
2533 int t;
2535 if (graph_alloc(ctx, &split, n, n_edge) < 0)
2536 goto error;
2537 if (copy_nodes(&split, graph, node_pred, data) < 0)
2538 goto error;
2539 if (graph_init_table(ctx, &split) < 0)
2540 goto error;
2541 for (t = 0; t <= isl_edge_last; ++t)
2542 split.max_edge[t] = graph->max_edge[t];
2543 if (graph_init_edge_tables(ctx, &split) < 0)
2544 goto error;
2545 if (copy_edges(ctx, &split, graph, edge_pred, data) < 0)
2546 goto error;
2547 split.n_row = graph->n_row;
2548 split.max_row = graph->max_row;
2549 split.n_total_row = graph->n_total_row;
2550 split.n_band = graph->n_band;
2551 split.band_start = graph->band_start;
2553 if (wcc && compute_schedule_wcc(ctx, &split) < 0)
2554 goto error;
2555 if (!wcc && compute_schedule(ctx, &split) < 0)
2556 goto error;
2558 copy_schedule(graph, &split, node_pred, data);
2560 graph_free(ctx, &split);
2561 return 0;
2562 error:
2563 graph_free(ctx, &split);
2564 return -1;
2567 static int node_scc_exactly(struct isl_sched_node *node, int scc)
2569 return node->scc == scc;
2572 static int node_scc_at_most(struct isl_sched_node *node, int scc)
2574 return node->scc <= scc;
2577 static int node_scc_at_least(struct isl_sched_node *node, int scc)
2579 return node->scc >= scc;
2582 static int edge_scc_exactly(struct isl_sched_edge *edge, int scc)
2584 return edge->src->scc == scc && edge->dst->scc == scc;
2587 static int edge_dst_scc_at_most(struct isl_sched_edge *edge, int scc)
2589 return edge->dst->scc <= scc;
2592 static int edge_src_scc_at_least(struct isl_sched_edge *edge, int scc)
2594 return edge->src->scc >= scc;
2597 /* Pad the schedules of all nodes with zero rows such that in the end
2598 * they all have graph->n_total_row rows.
2599 * The extra rows don't belong to any band, so they get assigned band number -1.
2601 static int pad_schedule(struct isl_sched_graph *graph)
2603 int i, j;
2605 for (i = 0; i < graph->n; ++i) {
2606 struct isl_sched_node *node = &graph->node[i];
2607 int row = isl_mat_rows(node->sched);
2608 if (graph->n_total_row > row) {
2609 isl_map_free(node->sched_map);
2610 node->sched_map = NULL;
2612 node->sched = isl_mat_add_zero_rows(node->sched,
2613 graph->n_total_row - row);
2614 if (!node->sched)
2615 return -1;
2616 for (j = row; j < graph->n_total_row; ++j)
2617 node->band[j] = -1;
2620 return 0;
2623 /* Reset the current band by dropping all its schedule rows.
2625 static int reset_band(struct isl_sched_graph *graph)
2627 int i;
2628 int drop;
2630 drop = graph->n_total_row - graph->band_start;
2631 graph->n_total_row -= drop;
2632 graph->n_row -= drop;
2634 for (i = 0; i < graph->n; ++i) {
2635 struct isl_sched_node *node = &graph->node[i];
2637 isl_map_free(node->sched_map);
2638 node->sched_map = NULL;
2640 node->sched = isl_mat_drop_rows(node->sched,
2641 graph->band_start, drop);
2643 if (!node->sched)
2644 return -1;
2647 return 0;
2650 /* Split the current graph into two parts and compute a schedule for each
2651 * part individually. In particular, one part consists of all SCCs up
2652 * to and including graph->src_scc, while the other part contains the other
2653 * SCCS.
2655 * The split is enforced in the schedule by constant rows with two different
2656 * values (0 and 1). These constant rows replace the previously computed rows
2657 * in the current band.
2658 * It would be possible to reuse them as the first rows in the next
2659 * band, but recomputing them may result in better rows as we are looking
2660 * at a smaller part of the dependence graph.
2662 * Since we do not enforce coincidence, we conservatively mark the
2663 * splitting row as not coincident.
2665 * The band_id of the second group is set to n, where n is the number
2666 * of nodes in the first group. This ensures that the band_ids over
2667 * the two groups remain disjoint, even if either or both of the two
2668 * groups contain independent components.
2670 static int compute_split_schedule(isl_ctx *ctx, struct isl_sched_graph *graph)
2672 int i, j, n, e1, e2;
2673 int n_total_row, orig_total_row;
2674 int n_band, orig_band;
2676 if (graph->n_total_row >= graph->max_row)
2677 isl_die(ctx, isl_error_internal,
2678 "too many schedule rows", return -1);
2680 if (reset_band(graph) < 0)
2681 return -1;
2683 n = 0;
2684 for (i = 0; i < graph->n; ++i) {
2685 struct isl_sched_node *node = &graph->node[i];
2686 int row = isl_mat_rows(node->sched);
2687 int cols = isl_mat_cols(node->sched);
2688 int before = node->scc <= graph->src_scc;
2690 if (before)
2691 n++;
2693 isl_map_free(node->sched_map);
2694 node->sched_map = NULL;
2695 node->sched = isl_mat_add_rows(node->sched, 1);
2696 if (!node->sched)
2697 return -1;
2698 node->sched = isl_mat_set_element_si(node->sched, row, 0,
2699 !before);
2700 for (j = 1; j < cols; ++j)
2701 node->sched = isl_mat_set_element_si(node->sched,
2702 row, j, 0);
2703 node->band[graph->n_total_row] = graph->n_band;
2704 node->coincident[graph->n_total_row] = 0;
2707 e1 = e2 = 0;
2708 for (i = 0; i < graph->n_edge; ++i) {
2709 if (graph->edge[i].dst->scc <= graph->src_scc)
2710 e1++;
2711 if (graph->edge[i].src->scc > graph->src_scc)
2712 e2++;
2715 graph->n_total_row++;
2716 next_band(graph);
2718 for (i = 0; i < graph->n; ++i) {
2719 struct isl_sched_node *node = &graph->node[i];
2720 if (node->scc > graph->src_scc)
2721 node->band_id[graph->n_band] = n;
2724 orig_total_row = graph->n_total_row;
2725 orig_band = graph->n_band;
2726 if (compute_sub_schedule(ctx, graph, n, e1,
2727 &node_scc_at_most, &edge_dst_scc_at_most,
2728 graph->src_scc, 0) < 0)
2729 return -1;
2730 n_total_row = graph->n_total_row;
2731 graph->n_total_row = orig_total_row;
2732 n_band = graph->n_band;
2733 graph->n_band = orig_band;
2734 if (compute_sub_schedule(ctx, graph, graph->n - n, e2,
2735 &node_scc_at_least, &edge_src_scc_at_least,
2736 graph->src_scc + 1, 0) < 0)
2737 return -1;
2738 if (n_total_row > graph->n_total_row)
2739 graph->n_total_row = n_total_row;
2740 if (n_band > graph->n_band)
2741 graph->n_band = n_band;
2743 return pad_schedule(graph);
2746 /* Compute the next band of the schedule after updating the dependence
2747 * relations based on the the current schedule.
2749 static int compute_next_band(isl_ctx *ctx, struct isl_sched_graph *graph)
2751 if (update_edges(ctx, graph) < 0)
2752 return -1;
2753 next_band(graph);
2755 return compute_schedule(ctx, graph);
2758 /* Add constraints to graph->lp that force the dependence "map" (which
2759 * is part of the dependence relation of "edge")
2760 * to be respected and attempt to carry it, where the edge is one from
2761 * a node j to itself. "pos" is the sequence number of the given map.
2762 * That is, add constraints that enforce
2764 * (c_j_0 + c_j_n n + c_j_x y) - (c_j_0 + c_j_n n + c_j_x x)
2765 * = c_j_x (y - x) >= e_i
2767 * for each (x,y) in R.
2768 * We obtain general constraints on coefficients (c_0, c_n, c_x)
2769 * of valid constraints for (y - x) and then plug in (-e_i, 0, c_j_x),
2770 * with each coefficient in c_j_x represented as a pair of non-negative
2771 * coefficients.
2773 static int add_intra_constraints(struct isl_sched_graph *graph,
2774 struct isl_sched_edge *edge, __isl_take isl_map *map, int pos)
2776 unsigned total;
2777 isl_ctx *ctx = isl_map_get_ctx(map);
2778 isl_space *dim;
2779 isl_dim_map *dim_map;
2780 isl_basic_set *coef;
2781 struct isl_sched_node *node = edge->src;
2783 coef = intra_coefficients(graph, map);
2784 if (!coef)
2785 return -1;
2787 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
2789 total = isl_basic_set_total_dim(graph->lp);
2790 dim_map = isl_dim_map_alloc(ctx, total);
2791 isl_dim_map_range(dim_map, 3 + pos, 0, 0, 0, 1, -1);
2792 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 1, 2,
2793 isl_space_dim(dim, isl_dim_set), 1,
2794 node->nvar, -1);
2795 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 2, 2,
2796 isl_space_dim(dim, isl_dim_set), 1,
2797 node->nvar, 1);
2798 graph->lp = isl_basic_set_extend_constraints(graph->lp,
2799 coef->n_eq, coef->n_ineq);
2800 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
2801 coef, dim_map);
2802 isl_space_free(dim);
2804 return 0;
2807 /* Add constraints to graph->lp that force the dependence "map" (which
2808 * is part of the dependence relation of "edge")
2809 * to be respected and attempt to carry it, where the edge is one from
2810 * node j to node k. "pos" is the sequence number of the given map.
2811 * That is, add constraints that enforce
2813 * (c_k_0 + c_k_n n + c_k_x y) - (c_j_0 + c_j_n n + c_j_x x) >= e_i
2815 * for each (x,y) in R.
2816 * We obtain general constraints on coefficients (c_0, c_n, c_x)
2817 * of valid constraints for R and then plug in
2818 * (-e_i + c_k_0 - c_j_0, c_k_n - c_j_n, c_k_x - c_j_x)
2819 * with each coefficient (except e_i, c_k_0 and c_j_0)
2820 * represented as a pair of non-negative coefficients.
2822 static int add_inter_constraints(struct isl_sched_graph *graph,
2823 struct isl_sched_edge *edge, __isl_take isl_map *map, int pos)
2825 unsigned total;
2826 isl_ctx *ctx = isl_map_get_ctx(map);
2827 isl_space *dim;
2828 isl_dim_map *dim_map;
2829 isl_basic_set *coef;
2830 struct isl_sched_node *src = edge->src;
2831 struct isl_sched_node *dst = edge->dst;
2833 coef = inter_coefficients(graph, map);
2834 if (!coef)
2835 return -1;
2837 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
2839 total = isl_basic_set_total_dim(graph->lp);
2840 dim_map = isl_dim_map_alloc(ctx, total);
2842 isl_dim_map_range(dim_map, 3 + pos, 0, 0, 0, 1, -1);
2844 isl_dim_map_range(dim_map, dst->start, 0, 0, 0, 1, 1);
2845 isl_dim_map_range(dim_map, dst->start + 1, 2, 1, 1, dst->nparam, -1);
2846 isl_dim_map_range(dim_map, dst->start + 2, 2, 1, 1, dst->nparam, 1);
2847 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 1, 2,
2848 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
2849 dst->nvar, -1);
2850 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 2, 2,
2851 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
2852 dst->nvar, 1);
2854 isl_dim_map_range(dim_map, src->start, 0, 0, 0, 1, -1);
2855 isl_dim_map_range(dim_map, src->start + 1, 2, 1, 1, src->nparam, 1);
2856 isl_dim_map_range(dim_map, src->start + 2, 2, 1, 1, src->nparam, -1);
2857 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 1, 2,
2858 isl_space_dim(dim, isl_dim_set), 1,
2859 src->nvar, 1);
2860 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 2, 2,
2861 isl_space_dim(dim, isl_dim_set), 1,
2862 src->nvar, -1);
2864 graph->lp = isl_basic_set_extend_constraints(graph->lp,
2865 coef->n_eq, coef->n_ineq);
2866 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
2867 coef, dim_map);
2868 isl_space_free(dim);
2870 return 0;
2873 /* Add constraints to graph->lp that force all (conditional) validity
2874 * dependences to be respected and attempt to carry them.
2876 static int add_all_constraints(struct isl_sched_graph *graph)
2878 int i, j;
2879 int pos;
2881 pos = 0;
2882 for (i = 0; i < graph->n_edge; ++i) {
2883 struct isl_sched_edge *edge= &graph->edge[i];
2885 if (!edge->validity && !edge->conditional_validity)
2886 continue;
2888 for (j = 0; j < edge->map->n; ++j) {
2889 isl_basic_map *bmap;
2890 isl_map *map;
2892 bmap = isl_basic_map_copy(edge->map->p[j]);
2893 map = isl_map_from_basic_map(bmap);
2895 if (edge->src == edge->dst &&
2896 add_intra_constraints(graph, edge, map, pos) < 0)
2897 return -1;
2898 if (edge->src != edge->dst &&
2899 add_inter_constraints(graph, edge, map, pos) < 0)
2900 return -1;
2901 ++pos;
2905 return 0;
2908 /* Count the number of equality and inequality constraints
2909 * that will be added to the carry_lp problem.
2910 * We count each edge exactly once.
2912 static int count_all_constraints(struct isl_sched_graph *graph,
2913 int *n_eq, int *n_ineq)
2915 int i, j;
2917 *n_eq = *n_ineq = 0;
2918 for (i = 0; i < graph->n_edge; ++i) {
2919 struct isl_sched_edge *edge= &graph->edge[i];
2920 for (j = 0; j < edge->map->n; ++j) {
2921 isl_basic_map *bmap;
2922 isl_map *map;
2924 bmap = isl_basic_map_copy(edge->map->p[j]);
2925 map = isl_map_from_basic_map(bmap);
2927 if (count_map_constraints(graph, edge, map,
2928 n_eq, n_ineq, 1, 0) < 0)
2929 return -1;
2933 return 0;
2936 /* Construct an LP problem for finding schedule coefficients
2937 * such that the schedule carries as many dependences as possible.
2938 * In particular, for each dependence i, we bound the dependence distance
2939 * from below by e_i, with 0 <= e_i <= 1 and then maximize the sum
2940 * of all e_i's. Dependence with e_i = 0 in the solution are simply
2941 * respected, while those with e_i > 0 (in practice e_i = 1) are carried.
2942 * Note that if the dependence relation is a union of basic maps,
2943 * then we have to consider each basic map individually as it may only
2944 * be possible to carry the dependences expressed by some of those
2945 * basic maps and not all off them.
2946 * Below, we consider each of those basic maps as a separate "edge".
2948 * All variables of the LP are non-negative. The actual coefficients
2949 * may be negative, so each coefficient is represented as the difference
2950 * of two non-negative variables. The negative part always appears
2951 * immediately before the positive part.
2952 * Other than that, the variables have the following order
2954 * - sum of (1 - e_i) over all edges
2955 * - sum of positive and negative parts of all c_n coefficients
2956 * (unconstrained when computing non-parametric schedules)
2957 * - sum of positive and negative parts of all c_x coefficients
2958 * - for each edge
2959 * - e_i
2960 * - for each node
2961 * - c_i_0
2962 * - positive and negative parts of c_i_n (if parametric)
2963 * - positive and negative parts of c_i_x
2965 * The constraints are those from the (validity) edges plus three equalities
2966 * to express the sums and n_edge inequalities to express e_i <= 1.
2968 static int setup_carry_lp(isl_ctx *ctx, struct isl_sched_graph *graph)
2970 int i, j;
2971 int k;
2972 isl_space *dim;
2973 unsigned total;
2974 int n_eq, n_ineq;
2975 int n_edge;
2977 n_edge = 0;
2978 for (i = 0; i < graph->n_edge; ++i)
2979 n_edge += graph->edge[i].map->n;
2981 total = 3 + n_edge;
2982 for (i = 0; i < graph->n; ++i) {
2983 struct isl_sched_node *node = &graph->node[graph->sorted[i]];
2984 node->start = total;
2985 total += 1 + 2 * (node->nparam + node->nvar);
2988 if (count_all_constraints(graph, &n_eq, &n_ineq) < 0)
2989 return -1;
2990 if (count_bound_coefficient_constraints(ctx, graph, &n_eq, &n_ineq) < 0)
2991 return -1;
2993 dim = isl_space_set_alloc(ctx, 0, total);
2994 isl_basic_set_free(graph->lp);
2995 n_eq += 3;
2996 n_ineq += n_edge;
2997 graph->lp = isl_basic_set_alloc_space(dim, 0, n_eq, n_ineq);
2998 graph->lp = isl_basic_set_set_rational(graph->lp);
3000 k = isl_basic_set_alloc_equality(graph->lp);
3001 if (k < 0)
3002 return -1;
3003 isl_seq_clr(graph->lp->eq[k], 1 + total);
3004 isl_int_set_si(graph->lp->eq[k][0], -n_edge);
3005 isl_int_set_si(graph->lp->eq[k][1], 1);
3006 for (i = 0; i < n_edge; ++i)
3007 isl_int_set_si(graph->lp->eq[k][4 + i], 1);
3009 k = isl_basic_set_alloc_equality(graph->lp);
3010 if (k < 0)
3011 return -1;
3012 isl_seq_clr(graph->lp->eq[k], 1 + total);
3013 isl_int_set_si(graph->lp->eq[k][2], -1);
3014 for (i = 0; i < graph->n; ++i) {
3015 int pos = 1 + graph->node[i].start + 1;
3017 for (j = 0; j < 2 * graph->node[i].nparam; ++j)
3018 isl_int_set_si(graph->lp->eq[k][pos + j], 1);
3021 k = isl_basic_set_alloc_equality(graph->lp);
3022 if (k < 0)
3023 return -1;
3024 isl_seq_clr(graph->lp->eq[k], 1 + total);
3025 isl_int_set_si(graph->lp->eq[k][3], -1);
3026 for (i = 0; i < graph->n; ++i) {
3027 struct isl_sched_node *node = &graph->node[i];
3028 int pos = 1 + node->start + 1 + 2 * node->nparam;
3030 for (j = 0; j < 2 * node->nvar; ++j)
3031 isl_int_set_si(graph->lp->eq[k][pos + j], 1);
3034 for (i = 0; i < n_edge; ++i) {
3035 k = isl_basic_set_alloc_inequality(graph->lp);
3036 if (k < 0)
3037 return -1;
3038 isl_seq_clr(graph->lp->ineq[k], 1 + total);
3039 isl_int_set_si(graph->lp->ineq[k][4 + i], -1);
3040 isl_int_set_si(graph->lp->ineq[k][0], 1);
3043 if (add_bound_coefficient_constraints(ctx, graph) < 0)
3044 return -1;
3045 if (add_all_constraints(graph) < 0)
3046 return -1;
3048 return 0;
3051 /* If the schedule_split_scaled option is set and if the linear
3052 * parts of the scheduling rows for all nodes in the graphs have
3053 * non-trivial common divisor, then split off the constant term
3054 * from the linear part.
3055 * The constant term is then placed in a separate band and
3056 * the linear part is reduced.
3058 static int split_scaled(isl_ctx *ctx, struct isl_sched_graph *graph)
3060 int i;
3061 int row;
3062 isl_int gcd, gcd_i;
3064 if (!ctx->opt->schedule_split_scaled)
3065 return 0;
3066 if (graph->n <= 1)
3067 return 0;
3069 if (graph->n_total_row >= graph->max_row)
3070 isl_die(ctx, isl_error_internal,
3071 "too many schedule rows", return -1);
3073 isl_int_init(gcd);
3074 isl_int_init(gcd_i);
3076 isl_int_set_si(gcd, 0);
3078 row = isl_mat_rows(graph->node[0].sched) - 1;
3080 for (i = 0; i < graph->n; ++i) {
3081 struct isl_sched_node *node = &graph->node[i];
3082 int cols = isl_mat_cols(node->sched);
3084 isl_seq_gcd(node->sched->row[row] + 1, cols - 1, &gcd_i);
3085 isl_int_gcd(gcd, gcd, gcd_i);
3088 isl_int_clear(gcd_i);
3090 if (isl_int_cmp_si(gcd, 1) <= 0) {
3091 isl_int_clear(gcd);
3092 return 0;
3095 next_band(graph);
3097 for (i = 0; i < graph->n; ++i) {
3098 struct isl_sched_node *node = &graph->node[i];
3100 isl_map_free(node->sched_map);
3101 node->sched_map = NULL;
3102 node->sched = isl_mat_add_zero_rows(node->sched, 1);
3103 if (!node->sched)
3104 goto error;
3105 isl_int_fdiv_r(node->sched->row[row + 1][0],
3106 node->sched->row[row][0], gcd);
3107 isl_int_fdiv_q(node->sched->row[row][0],
3108 node->sched->row[row][0], gcd);
3109 isl_int_mul(node->sched->row[row][0],
3110 node->sched->row[row][0], gcd);
3111 node->sched = isl_mat_scale_down_row(node->sched, row, gcd);
3112 if (!node->sched)
3113 goto error;
3114 node->band[graph->n_total_row] = graph->n_band;
3117 graph->n_total_row++;
3119 isl_int_clear(gcd);
3120 return 0;
3121 error:
3122 isl_int_clear(gcd);
3123 return -1;
3126 static int compute_component_schedule(isl_ctx *ctx,
3127 struct isl_sched_graph *graph);
3129 /* Is the schedule row "sol" trivial on node "node"?
3130 * That is, is the solution zero on the dimensions orthogonal to
3131 * the previously found solutions?
3132 * Return 1 if the solution is trivial, 0 if it is not and -1 on error.
3134 * Each coefficient is represented as the difference between
3135 * two non-negative values in "sol". "sol" has been computed
3136 * in terms of the original iterators (i.e., without use of cmap).
3137 * We construct the schedule row s and write it as a linear
3138 * combination of (linear combinations of) previously computed schedule rows.
3139 * s = Q c or c = U s.
3140 * If the final entries of c are all zero, then the solution is trivial.
3142 static int is_trivial(struct isl_sched_node *node, __isl_keep isl_vec *sol)
3144 int i;
3145 int pos;
3146 int trivial;
3147 isl_ctx *ctx;
3148 isl_vec *node_sol;
3150 if (!sol)
3151 return -1;
3152 if (node->nvar == node->rank)
3153 return 0;
3155 ctx = isl_vec_get_ctx(sol);
3156 node_sol = isl_vec_alloc(ctx, node->nvar);
3157 if (!node_sol)
3158 return -1;
3160 pos = 1 + node->start + 1 + 2 * node->nparam;
3162 for (i = 0; i < node->nvar; ++i)
3163 isl_int_sub(node_sol->el[i],
3164 sol->el[pos + 2 * i + 1], sol->el[pos + 2 * i]);
3166 node_sol = isl_mat_vec_product(isl_mat_copy(node->cinv), node_sol);
3168 if (!node_sol)
3169 return -1;
3171 trivial = isl_seq_first_non_zero(node_sol->el + node->rank,
3172 node->nvar - node->rank) == -1;
3174 isl_vec_free(node_sol);
3176 return trivial;
3179 /* Is the schedule row "sol" trivial on any node where it should
3180 * not be trivial?
3181 * "sol" has been computed in terms of the original iterators
3182 * (i.e., without use of cmap).
3183 * Return 1 if any solution is trivial, 0 if they are not and -1 on error.
3185 static int is_any_trivial(struct isl_sched_graph *graph,
3186 __isl_keep isl_vec *sol)
3188 int i;
3190 for (i = 0; i < graph->n; ++i) {
3191 struct isl_sched_node *node = &graph->node[i];
3192 int trivial;
3194 if (!needs_row(graph, node))
3195 continue;
3196 trivial = is_trivial(node, sol);
3197 if (trivial < 0 || trivial)
3198 return trivial;
3201 return 0;
3204 /* Construct a schedule row for each node such that as many dependences
3205 * as possible are carried and then continue with the next band.
3207 * If the computed schedule row turns out to be trivial on one or
3208 * more nodes where it should not be trivial, then we throw it away
3209 * and try again on each component separately.
3211 static int carry_dependences(isl_ctx *ctx, struct isl_sched_graph *graph)
3213 int i;
3214 int n_edge;
3215 int trivial;
3216 isl_vec *sol;
3217 isl_basic_set *lp;
3219 n_edge = 0;
3220 for (i = 0; i < graph->n_edge; ++i)
3221 n_edge += graph->edge[i].map->n;
3223 if (setup_carry_lp(ctx, graph) < 0)
3224 return -1;
3226 lp = isl_basic_set_copy(graph->lp);
3227 sol = isl_tab_basic_set_non_neg_lexmin(lp);
3228 if (!sol)
3229 return -1;
3231 if (sol->size == 0) {
3232 isl_vec_free(sol);
3233 isl_die(ctx, isl_error_internal,
3234 "error in schedule construction", return -1);
3237 isl_int_divexact(sol->el[1], sol->el[1], sol->el[0]);
3238 if (isl_int_cmp_si(sol->el[1], n_edge) >= 0) {
3239 isl_vec_free(sol);
3240 isl_die(ctx, isl_error_unknown,
3241 "unable to carry dependences", return -1);
3244 trivial = is_any_trivial(graph, sol);
3245 if (trivial < 0) {
3246 sol = isl_vec_free(sol);
3247 } else if (trivial) {
3248 isl_vec_free(sol);
3249 if (graph->scc > 1)
3250 return compute_component_schedule(ctx, graph);
3251 isl_die(ctx, isl_error_unknown,
3252 "unable to construct non-trivial solution", return -1);
3255 if (update_schedule(graph, sol, 0, 0) < 0)
3256 return -1;
3258 if (split_scaled(ctx, graph) < 0)
3259 return -1;
3261 return compute_next_band(ctx, graph);
3264 /* Are there any (non-empty) (conditional) validity edges in the graph?
3266 static int has_validity_edges(struct isl_sched_graph *graph)
3268 int i;
3270 for (i = 0; i < graph->n_edge; ++i) {
3271 int empty;
3273 empty = isl_map_plain_is_empty(graph->edge[i].map);
3274 if (empty < 0)
3275 return -1;
3276 if (empty)
3277 continue;
3278 if (graph->edge[i].validity ||
3279 graph->edge[i].conditional_validity)
3280 return 1;
3283 return 0;
3286 /* Should we apply a Feautrier step?
3287 * That is, did the user request the Feautrier algorithm and are
3288 * there any validity dependences (left)?
3290 static int need_feautrier_step(isl_ctx *ctx, struct isl_sched_graph *graph)
3292 if (ctx->opt->schedule_algorithm != ISL_SCHEDULE_ALGORITHM_FEAUTRIER)
3293 return 0;
3295 return has_validity_edges(graph);
3298 /* Compute a schedule for a connected dependence graph using Feautrier's
3299 * multi-dimensional scheduling algorithm.
3300 * The original algorithm is described in [1].
3301 * The main idea is to minimize the number of scheduling dimensions, by
3302 * trying to satisfy as many dependences as possible per scheduling dimension.
3304 * [1] P. Feautrier, Some Efficient Solutions to the Affine Scheduling
3305 * Problem, Part II: Multi-Dimensional Time.
3306 * In Intl. Journal of Parallel Programming, 1992.
3308 static int compute_schedule_wcc_feautrier(isl_ctx *ctx,
3309 struct isl_sched_graph *graph)
3311 return carry_dependences(ctx, graph);
3314 /* Turn off the "local" bit on all (condition) edges.
3316 static void clear_local_edges(struct isl_sched_graph *graph)
3318 int i;
3320 for (i = 0; i < graph->n_edge; ++i)
3321 if (graph->edge[i].condition)
3322 graph->edge[i].local = 0;
3325 /* Does "graph" have both condition and conditional validity edges?
3327 static int need_condition_check(struct isl_sched_graph *graph)
3329 int i;
3330 int any_condition = 0;
3331 int any_conditional_validity = 0;
3333 for (i = 0; i < graph->n_edge; ++i) {
3334 if (graph->edge[i].condition)
3335 any_condition = 1;
3336 if (graph->edge[i].conditional_validity)
3337 any_conditional_validity = 1;
3340 return any_condition && any_conditional_validity;
3343 /* Does "graph" contain any coincidence edge?
3345 static int has_any_coincidence(struct isl_sched_graph *graph)
3347 int i;
3349 for (i = 0; i < graph->n_edge; ++i)
3350 if (graph->edge[i].coincidence)
3351 return 1;
3353 return 0;
3356 /* Extract the final schedule row as a map with the iteration domain
3357 * of "node" as domain.
3359 static __isl_give isl_map *final_row(struct isl_sched_node *node)
3361 isl_local_space *ls;
3362 isl_aff *aff;
3363 int row;
3365 row = isl_mat_rows(node->sched) - 1;
3366 ls = isl_local_space_from_space(isl_space_copy(node->dim));
3367 aff = extract_schedule_row(ls, node, row);
3368 return isl_map_from_aff(aff);
3371 /* Is the conditional validity dependence in the edge with index "edge_index"
3372 * violated by the latest (i.e., final) row of the schedule?
3373 * That is, is i scheduled after j
3374 * for any conditional validity dependence i -> j?
3376 static int is_violated(struct isl_sched_graph *graph, int edge_index)
3378 isl_map *src_sched, *dst_sched, *map;
3379 struct isl_sched_edge *edge = &graph->edge[edge_index];
3380 int empty;
3382 src_sched = final_row(edge->src);
3383 dst_sched = final_row(edge->dst);
3384 map = isl_map_copy(edge->map);
3385 map = isl_map_apply_domain(map, src_sched);
3386 map = isl_map_apply_range(map, dst_sched);
3387 map = isl_map_order_gt(map, isl_dim_in, 0, isl_dim_out, 0);
3388 empty = isl_map_is_empty(map);
3389 isl_map_free(map);
3391 if (empty < 0)
3392 return -1;
3394 return !empty;
3397 /* Does the domain of "umap" intersect "uset"?
3399 static int domain_intersects(__isl_keep isl_union_map *umap,
3400 __isl_keep isl_union_set *uset)
3402 int empty;
3404 umap = isl_union_map_copy(umap);
3405 umap = isl_union_map_intersect_domain(umap, isl_union_set_copy(uset));
3406 empty = isl_union_map_is_empty(umap);
3407 isl_union_map_free(umap);
3409 return empty < 0 ? -1 : !empty;
3412 /* Does the range of "umap" intersect "uset"?
3414 static int range_intersects(__isl_keep isl_union_map *umap,
3415 __isl_keep isl_union_set *uset)
3417 int empty;
3419 umap = isl_union_map_copy(umap);
3420 umap = isl_union_map_intersect_range(umap, isl_union_set_copy(uset));
3421 empty = isl_union_map_is_empty(umap);
3422 isl_union_map_free(umap);
3424 return empty < 0 ? -1 : !empty;
3427 /* Are the condition dependences of "edge" local with respect to
3428 * the current schedule?
3430 * That is, are domain and range of the condition dependences mapped
3431 * to the same point?
3433 * In other words, is the condition false?
3435 static int is_condition_false(struct isl_sched_edge *edge)
3437 isl_union_map *umap;
3438 isl_map *map, *sched, *test;
3439 int local;
3441 umap = isl_union_map_copy(edge->tagged_condition);
3442 umap = isl_union_map_zip(umap);
3443 umap = isl_union_set_unwrap(isl_union_map_domain(umap));
3444 map = isl_map_from_union_map(umap);
3446 sched = node_extract_schedule(edge->src);
3447 map = isl_map_apply_domain(map, sched);
3448 sched = node_extract_schedule(edge->dst);
3449 map = isl_map_apply_range(map, sched);
3451 test = isl_map_identity(isl_map_get_space(map));
3452 local = isl_map_is_subset(map, test);
3453 isl_map_free(map);
3454 isl_map_free(test);
3456 return local;
3459 /* Does "graph" have any satisfied condition edges that
3460 * are adjacent to the conditional validity constraint with
3461 * domain "conditional_source" and range "conditional_sink"?
3463 * A satisfied condition is one that is not local.
3464 * If a condition was forced to be local already (i.e., marked as local)
3465 * then there is no need to check if it is in fact local.
3467 * Additionally, mark all adjacent condition edges found as local.
3469 static int has_adjacent_true_conditions(struct isl_sched_graph *graph,
3470 __isl_keep isl_union_set *conditional_source,
3471 __isl_keep isl_union_set *conditional_sink)
3473 int i;
3474 int any = 0;
3476 for (i = 0; i < graph->n_edge; ++i) {
3477 int adjacent, local;
3478 isl_union_map *condition;
3480 if (!graph->edge[i].condition)
3481 continue;
3482 if (graph->edge[i].local)
3483 continue;
3485 condition = graph->edge[i].tagged_condition;
3486 adjacent = domain_intersects(condition, conditional_sink);
3487 if (adjacent >= 0 && !adjacent)
3488 adjacent = range_intersects(condition,
3489 conditional_source);
3490 if (adjacent < 0)
3491 return -1;
3492 if (!adjacent)
3493 continue;
3495 graph->edge[i].local = 1;
3497 local = is_condition_false(&graph->edge[i]);
3498 if (local < 0)
3499 return -1;
3500 if (!local)
3501 any = 1;
3504 return any;
3507 /* Are there any violated conditional validity dependences with
3508 * adjacent condition dependences that are not local with respect
3509 * to the current schedule?
3510 * That is, is the conditional validity constraint violated?
3512 * Additionally, mark all those adjacent condition dependences as local.
3513 * We also mark those adjacent condition dependences that were not marked
3514 * as local before, but just happened to be local already. This ensures
3515 * that they remain local if the schedule is recomputed.
3517 * We first collect domain and range of all violated conditional validity
3518 * dependences and then check if there are any adjacent non-local
3519 * condition dependences.
3521 static int has_violated_conditional_constraint(isl_ctx *ctx,
3522 struct isl_sched_graph *graph)
3524 int i;
3525 int any = 0;
3526 isl_union_set *source, *sink;
3528 source = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
3529 sink = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
3530 for (i = 0; i < graph->n_edge; ++i) {
3531 isl_union_set *uset;
3532 isl_union_map *umap;
3533 int violated;
3535 if (!graph->edge[i].conditional_validity)
3536 continue;
3538 violated = is_violated(graph, i);
3539 if (violated < 0)
3540 goto error;
3541 if (!violated)
3542 continue;
3544 any = 1;
3546 umap = isl_union_map_copy(graph->edge[i].tagged_validity);
3547 uset = isl_union_map_domain(umap);
3548 source = isl_union_set_union(source, uset);
3549 source = isl_union_set_coalesce(source);
3551 umap = isl_union_map_copy(graph->edge[i].tagged_validity);
3552 uset = isl_union_map_range(umap);
3553 sink = isl_union_set_union(sink, uset);
3554 sink = isl_union_set_coalesce(sink);
3557 if (any)
3558 any = has_adjacent_true_conditions(graph, source, sink);
3560 isl_union_set_free(source);
3561 isl_union_set_free(sink);
3562 return any;
3563 error:
3564 isl_union_set_free(source);
3565 isl_union_set_free(sink);
3566 return -1;
3569 /* Compute a schedule for a connected dependence graph.
3570 * We try to find a sequence of as many schedule rows as possible that result
3571 * in non-negative dependence distances (independent of the previous rows
3572 * in the sequence, i.e., such that the sequence is tilable), with as
3573 * many of the initial rows as possible satisfying the coincidence constraints.
3574 * If we can't find any more rows we either
3575 * - split between SCCs and start over (assuming we found an interesting
3576 * pair of SCCs between which to split)
3577 * - continue with the next band (assuming the current band has at least
3578 * one row)
3579 * - try to carry as many dependences as possible and continue with the next
3580 * band
3582 * If Feautrier's algorithm is selected, we first recursively try to satisfy
3583 * as many validity dependences as possible. When all validity dependences
3584 * are satisfied we extend the schedule to a full-dimensional schedule.
3586 * If we manage to complete the schedule, we finish off by topologically
3587 * sorting the statements based on the remaining dependences.
3589 * If ctx->opt->schedule_outer_coincidence is set, then we force the
3590 * outermost dimension to satisfy the coincidence constraints. If this
3591 * turns out to be impossible, we fall back on the general scheme above
3592 * and try to carry as many dependences as possible.
3594 * If "graph" contains both condition and conditional validity dependences,
3595 * then we need to check that that the conditional schedule constraint
3596 * is satisfied, i.e., there are no violated conditional validity dependences
3597 * that are adjacent to any non-local condition dependences.
3598 * If there are, then we mark all those adjacent condition dependences
3599 * as local and recompute the current band. Those dependences that
3600 * are marked local will then be forced to be local.
3601 * The initial computation is performed with no dependences marked as local.
3602 * If we are lucky, then there will be no violated conditional validity
3603 * dependences adjacent to any non-local condition dependences.
3604 * Otherwise, we mark some additional condition dependences as local and
3605 * recompute. We continue this process until there are no violations left or
3606 * until we are no longer able to compute a schedule.
3607 * Since there are only a finite number of dependences,
3608 * there will only be a finite number of iterations.
3610 static int compute_schedule_wcc(isl_ctx *ctx, struct isl_sched_graph *graph)
3612 int has_coincidence;
3613 int use_coincidence;
3614 int force_coincidence = 0;
3615 int check_conditional;
3617 if (detect_sccs(ctx, graph) < 0)
3618 return -1;
3619 if (sort_sccs(graph) < 0)
3620 return -1;
3622 if (compute_maxvar(graph) < 0)
3623 return -1;
3625 if (need_feautrier_step(ctx, graph))
3626 return compute_schedule_wcc_feautrier(ctx, graph);
3628 clear_local_edges(graph);
3629 check_conditional = need_condition_check(graph);
3630 has_coincidence = has_any_coincidence(graph);
3632 if (ctx->opt->schedule_outer_coincidence)
3633 force_coincidence = 1;
3635 use_coincidence = has_coincidence;
3636 while (graph->n_row < graph->maxvar) {
3637 isl_vec *sol;
3638 int violated;
3639 int coincident;
3641 graph->src_scc = -1;
3642 graph->dst_scc = -1;
3644 if (setup_lp(ctx, graph, use_coincidence) < 0)
3645 return -1;
3646 sol = solve_lp(graph);
3647 if (!sol)
3648 return -1;
3649 if (sol->size == 0) {
3650 int empty = graph->n_total_row == graph->band_start;
3652 isl_vec_free(sol);
3653 if (use_coincidence && (!force_coincidence || !empty)) {
3654 use_coincidence = 0;
3655 continue;
3657 if (!ctx->opt->schedule_maximize_band_depth && !empty)
3658 return compute_next_band(ctx, graph);
3659 if (graph->src_scc >= 0)
3660 return compute_split_schedule(ctx, graph);
3661 if (!empty)
3662 return compute_next_band(ctx, graph);
3663 return carry_dependences(ctx, graph);
3665 coincident = !has_coincidence || use_coincidence;
3666 if (update_schedule(graph, sol, 1, coincident) < 0)
3667 return -1;
3669 if (!check_conditional)
3670 continue;
3671 violated = has_violated_conditional_constraint(ctx, graph);
3672 if (violated < 0)
3673 return -1;
3674 if (!violated)
3675 continue;
3676 if (reset_band(graph) < 0)
3677 return -1;
3678 use_coincidence = has_coincidence;
3681 if (graph->n_total_row > graph->band_start)
3682 next_band(graph);
3683 return sort_statements(ctx, graph);
3686 /* Add a row to the schedules that separates the SCCs and move
3687 * to the next band.
3689 static int split_on_scc(isl_ctx *ctx, struct isl_sched_graph *graph)
3691 int i;
3693 if (graph->n_total_row >= graph->max_row)
3694 isl_die(ctx, isl_error_internal,
3695 "too many schedule rows", return -1);
3697 for (i = 0; i < graph->n; ++i) {
3698 struct isl_sched_node *node = &graph->node[i];
3699 int row = isl_mat_rows(node->sched);
3701 isl_map_free(node->sched_map);
3702 node->sched_map = NULL;
3703 node->sched = isl_mat_add_zero_rows(node->sched, 1);
3704 node->sched = isl_mat_set_element_si(node->sched, row, 0,
3705 node->scc);
3706 if (!node->sched)
3707 return -1;
3708 node->band[graph->n_total_row] = graph->n_band;
3711 graph->n_total_row++;
3712 next_band(graph);
3714 return 0;
3717 /* Compute a schedule for each component (identified by node->scc)
3718 * of the dependence graph separately and then combine the results.
3719 * Depending on the setting of schedule_fuse, a component may be
3720 * either weakly or strongly connected.
3722 * The band_id is adjusted such that each component has a separate id.
3723 * Note that the band_id may have already been set to a value different
3724 * from zero by compute_split_schedule.
3726 static int compute_component_schedule(isl_ctx *ctx,
3727 struct isl_sched_graph *graph)
3729 int wcc, i;
3730 int n, n_edge;
3731 int n_total_row, orig_total_row;
3732 int n_band, orig_band;
3734 if (ctx->opt->schedule_fuse == ISL_SCHEDULE_FUSE_MIN ||
3735 ctx->opt->schedule_separate_components)
3736 if (split_on_scc(ctx, graph) < 0)
3737 return -1;
3739 n_total_row = 0;
3740 orig_total_row = graph->n_total_row;
3741 n_band = 0;
3742 orig_band = graph->n_band;
3743 for (i = 0; i < graph->n; ++i)
3744 graph->node[i].band_id[graph->n_band] += graph->node[i].scc;
3745 for (wcc = 0; wcc < graph->scc; ++wcc) {
3746 n = 0;
3747 for (i = 0; i < graph->n; ++i)
3748 if (graph->node[i].scc == wcc)
3749 n++;
3750 n_edge = 0;
3751 for (i = 0; i < graph->n_edge; ++i)
3752 if (graph->edge[i].src->scc == wcc &&
3753 graph->edge[i].dst->scc == wcc)
3754 n_edge++;
3756 if (compute_sub_schedule(ctx, graph, n, n_edge,
3757 &node_scc_exactly,
3758 &edge_scc_exactly, wcc, 1) < 0)
3759 return -1;
3760 if (graph->n_total_row > n_total_row)
3761 n_total_row = graph->n_total_row;
3762 graph->n_total_row = orig_total_row;
3763 if (graph->n_band > n_band)
3764 n_band = graph->n_band;
3765 graph->n_band = orig_band;
3768 graph->n_total_row = n_total_row;
3769 graph->n_band = n_band;
3771 return pad_schedule(graph);
3774 /* Compute a schedule for the given dependence graph.
3775 * We first check if the graph is connected (through validity and conditional
3776 * validity dependences) and, if not, compute a schedule
3777 * for each component separately.
3778 * If schedule_fuse is set to minimal fusion, then we check for strongly
3779 * connected components instead and compute a separate schedule for
3780 * each such strongly connected component.
3782 static int compute_schedule(isl_ctx *ctx, struct isl_sched_graph *graph)
3784 if (ctx->opt->schedule_fuse == ISL_SCHEDULE_FUSE_MIN) {
3785 if (detect_sccs(ctx, graph) < 0)
3786 return -1;
3787 } else {
3788 if (detect_wccs(ctx, graph) < 0)
3789 return -1;
3792 if (graph->scc > 1)
3793 return compute_component_schedule(ctx, graph);
3795 return compute_schedule_wcc(ctx, graph);
3798 /* Compute a schedule on sc->domain that respects the given schedule
3799 * constraints.
3801 * In particular, the schedule respects all the validity dependences.
3802 * If the default isl scheduling algorithm is used, it tries to minimize
3803 * the dependence distances over the proximity dependences.
3804 * If Feautrier's scheduling algorithm is used, the proximity dependence
3805 * distances are only minimized during the extension to a full-dimensional
3806 * schedule.
3808 * If there are any condition and conditional validity dependences,
3809 * then the conditional validity dependences may be violated inside
3810 * a tilable band, provided they have no adjacent non-local
3811 * condition dependences.
3813 __isl_give isl_schedule *isl_schedule_constraints_compute_schedule(
3814 __isl_take isl_schedule_constraints *sc)
3816 isl_ctx *ctx = isl_schedule_constraints_get_ctx(sc);
3817 struct isl_sched_graph graph = { 0 };
3818 isl_schedule *sched;
3819 struct isl_extract_edge_data data;
3820 enum isl_edge_type i;
3822 sc = isl_schedule_constraints_align_params(sc);
3823 if (!sc)
3824 return NULL;
3826 graph.n = isl_union_set_n_set(sc->domain);
3827 if (graph.n == 0)
3828 goto empty;
3829 if (graph_alloc(ctx, &graph, graph.n,
3830 isl_schedule_constraints_n_map(sc)) < 0)
3831 goto error;
3832 if (compute_max_row(&graph, sc->domain) < 0)
3833 goto error;
3834 graph.root = 1;
3835 graph.n = 0;
3836 if (isl_union_set_foreach_set(sc->domain, &extract_node, &graph) < 0)
3837 goto error;
3838 if (graph_init_table(ctx, &graph) < 0)
3839 goto error;
3840 for (i = isl_edge_first; i <= isl_edge_last; ++i)
3841 graph.max_edge[i] = isl_union_map_n_map(sc->constraint[i]);
3842 if (graph_init_edge_tables(ctx, &graph) < 0)
3843 goto error;
3844 graph.n_edge = 0;
3845 data.graph = &graph;
3846 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
3847 data.type = i;
3848 if (isl_union_map_foreach_map(sc->constraint[i],
3849 &extract_edge, &data) < 0)
3850 goto error;
3853 if (compute_schedule(ctx, &graph) < 0)
3854 goto error;
3856 empty:
3857 sched = extract_schedule(&graph, isl_union_set_get_space(sc->domain));
3859 graph_free(ctx, &graph);
3860 isl_schedule_constraints_free(sc);
3862 return sched;
3863 error:
3864 graph_free(ctx, &graph);
3865 isl_schedule_constraints_free(sc);
3866 return NULL;
3869 /* Compute a schedule for the given union of domains that respects
3870 * all the validity dependences and minimizes
3871 * the dependence distances over the proximity dependences.
3873 * This function is kept for backward compatibility.
3875 __isl_give isl_schedule *isl_union_set_compute_schedule(
3876 __isl_take isl_union_set *domain,
3877 __isl_take isl_union_map *validity,
3878 __isl_take isl_union_map *proximity)
3880 isl_schedule_constraints *sc;
3882 sc = isl_schedule_constraints_on_domain(domain);
3883 sc = isl_schedule_constraints_set_validity(sc, validity);
3884 sc = isl_schedule_constraints_set_proximity(sc, proximity);
3886 return isl_schedule_constraints_compute_schedule(sc);
3889 __isl_null isl_schedule *isl_schedule_free(__isl_take isl_schedule *sched)
3891 int i;
3892 if (!sched)
3893 return NULL;
3895 if (--sched->ref > 0)
3896 return NULL;
3898 for (i = 0; i < sched->n; ++i) {
3899 isl_multi_aff_free(sched->node[i].sched);
3900 free(sched->node[i].band_end);
3901 free(sched->node[i].band_id);
3902 free(sched->node[i].coincident);
3904 isl_space_free(sched->dim);
3905 isl_band_list_free(sched->band_forest);
3906 free(sched);
3907 return NULL;
3910 isl_ctx *isl_schedule_get_ctx(__isl_keep isl_schedule *schedule)
3912 return schedule ? isl_space_get_ctx(schedule->dim) : NULL;
3915 /* Set max_out to the maximal number of output dimensions over
3916 * all maps.
3918 static int update_max_out(__isl_take isl_map *map, void *user)
3920 int *max_out = user;
3921 int n_out = isl_map_dim(map, isl_dim_out);
3923 if (n_out > *max_out)
3924 *max_out = n_out;
3926 isl_map_free(map);
3927 return 0;
3930 /* Internal data structure for map_pad_range.
3932 * "max_out" is the maximal schedule dimension.
3933 * "res" collects the results.
3935 struct isl_pad_schedule_map_data {
3936 int max_out;
3937 isl_union_map *res;
3940 /* Pad the range of the given map with zeros to data->max_out and
3941 * then add the result to data->res.
3943 static int map_pad_range(__isl_take isl_map *map, void *user)
3945 struct isl_pad_schedule_map_data *data = user;
3946 int i;
3947 int n_out = isl_map_dim(map, isl_dim_out);
3949 map = isl_map_add_dims(map, isl_dim_out, data->max_out - n_out);
3950 for (i = n_out; i < data->max_out; ++i)
3951 map = isl_map_fix_si(map, isl_dim_out, i, 0);
3953 data->res = isl_union_map_add_map(data->res, map);
3954 if (!data->res)
3955 return -1;
3957 return 0;
3960 /* Pad the ranges of the maps in the union map with zeros such they all have
3961 * the same dimension.
3963 static __isl_give isl_union_map *pad_schedule_map(
3964 __isl_take isl_union_map *umap)
3966 struct isl_pad_schedule_map_data data;
3968 if (!umap)
3969 return NULL;
3970 if (isl_union_map_n_map(umap) <= 1)
3971 return umap;
3973 data.max_out = 0;
3974 if (isl_union_map_foreach_map(umap, &update_max_out, &data.max_out) < 0)
3975 return isl_union_map_free(umap);
3977 data.res = isl_union_map_empty(isl_union_map_get_space(umap));
3978 if (isl_union_map_foreach_map(umap, &map_pad_range, &data) < 0)
3979 data.res = isl_union_map_free(data.res);
3981 isl_union_map_free(umap);
3982 return data.res;
3985 /* Return an isl_union_map of the schedule. If we have already constructed
3986 * a band forest, then this band forest may have been modified so we need
3987 * to extract the isl_union_map from the forest rather than from
3988 * the originally computed schedule. This reconstructed schedule map
3989 * then needs to be padded with zeros to unify the schedule space
3990 * since the result of isl_band_list_get_suffix_schedule may not have
3991 * a unified schedule space.
3993 __isl_give isl_union_map *isl_schedule_get_map(__isl_keep isl_schedule *sched)
3995 int i;
3996 isl_union_map *umap;
3998 if (!sched)
3999 return NULL;
4001 if (sched->band_forest) {
4002 umap = isl_band_list_get_suffix_schedule(sched->band_forest);
4003 return pad_schedule_map(umap);
4006 umap = isl_union_map_empty(isl_space_copy(sched->dim));
4007 for (i = 0; i < sched->n; ++i) {
4008 isl_multi_aff *ma;
4010 ma = isl_multi_aff_copy(sched->node[i].sched);
4011 umap = isl_union_map_add_map(umap, isl_map_from_multi_aff(ma));
4014 return umap;
4017 static __isl_give isl_band_list *construct_band_list(
4018 __isl_keep isl_schedule *schedule, __isl_keep isl_band *parent,
4019 int band_nr, int *parent_active, int n_active);
4021 /* Construct an isl_band structure for the band in the given schedule
4022 * with sequence number band_nr for the n_active nodes marked by active.
4023 * If the nodes don't have a band with the given sequence number,
4024 * then a band without members is created.
4026 * Because of the way the schedule is constructed, we know that
4027 * the position of the band inside the schedule of a node is the same
4028 * for all active nodes.
4030 * The partial schedule for the band is created before the children
4031 * are created to that construct_band_list can refer to the partial
4032 * schedule of the parent.
4034 static __isl_give isl_band *construct_band(__isl_keep isl_schedule *schedule,
4035 __isl_keep isl_band *parent,
4036 int band_nr, int *active, int n_active)
4038 int i, j;
4039 isl_ctx *ctx = isl_schedule_get_ctx(schedule);
4040 isl_band *band;
4041 unsigned start, end;
4043 band = isl_band_alloc(ctx);
4044 if (!band)
4045 return NULL;
4047 band->schedule = schedule;
4048 band->parent = parent;
4050 for (i = 0; i < schedule->n; ++i)
4051 if (active[i])
4052 break;
4054 if (i >= schedule->n)
4055 isl_die(ctx, isl_error_internal,
4056 "band without active statements", goto error);
4058 start = band_nr ? schedule->node[i].band_end[band_nr - 1] : 0;
4059 end = band_nr < schedule->node[i].n_band ?
4060 schedule->node[i].band_end[band_nr] : start;
4061 band->n = end - start;
4063 band->coincident = isl_alloc_array(ctx, int, band->n);
4064 if (band->n && !band->coincident)
4065 goto error;
4067 for (j = 0; j < band->n; ++j)
4068 band->coincident[j] = schedule->node[i].coincident[start + j];
4070 band->pma = isl_union_pw_multi_aff_empty(isl_space_copy(schedule->dim));
4071 for (i = 0; i < schedule->n; ++i) {
4072 isl_multi_aff *ma;
4073 isl_pw_multi_aff *pma;
4074 unsigned n_out;
4076 if (!active[i])
4077 continue;
4079 ma = isl_multi_aff_copy(schedule->node[i].sched);
4080 n_out = isl_multi_aff_dim(ma, isl_dim_out);
4081 ma = isl_multi_aff_drop_dims(ma, isl_dim_out, end, n_out - end);
4082 ma = isl_multi_aff_drop_dims(ma, isl_dim_out, 0, start);
4083 pma = isl_pw_multi_aff_from_multi_aff(ma);
4084 band->pma = isl_union_pw_multi_aff_add_pw_multi_aff(band->pma,
4085 pma);
4087 if (!band->pma)
4088 goto error;
4090 for (i = 0; i < schedule->n; ++i)
4091 if (active[i] && schedule->node[i].n_band > band_nr + 1)
4092 break;
4094 if (i < schedule->n) {
4095 band->children = construct_band_list(schedule, band,
4096 band_nr + 1, active, n_active);
4097 if (!band->children)
4098 goto error;
4101 return band;
4102 error:
4103 isl_band_free(band);
4104 return NULL;
4107 /* Internal data structure used inside cmp_band and pw_multi_aff_extract_int.
4109 * r is set to a negative value if anything goes wrong.
4111 * c1 stores the result of extract_int.
4112 * c2 is a temporary value used inside cmp_band_in_ancestor.
4113 * t is a temporary value used inside extract_int.
4115 * first and equal are used inside extract_int.
4116 * first is set if we are looking at the first isl_multi_aff inside
4117 * the isl_union_pw_multi_aff.
4118 * equal is set if all the isl_multi_affs have been equal so far.
4120 struct isl_cmp_band_data {
4121 int r;
4123 int first;
4124 int equal;
4126 isl_int t;
4127 isl_int c1;
4128 isl_int c2;
4131 /* Check if "ma" assigns a constant value.
4132 * Note that this function is only called on isl_multi_affs
4133 * with a single output dimension.
4135 * If "ma" assigns a constant value then we compare it to data->c1
4136 * or assign it to data->c1 if this is the first isl_multi_aff we consider.
4137 * If "ma" does not assign a constant value or if it assigns a value
4138 * that is different from data->c1, then we set data->equal to zero
4139 * and terminate the check.
4141 static int multi_aff_extract_int(__isl_take isl_set *set,
4142 __isl_take isl_multi_aff *ma, void *user)
4144 isl_aff *aff;
4145 struct isl_cmp_band_data *data = user;
4147 aff = isl_multi_aff_get_aff(ma, 0);
4148 data->r = isl_aff_is_cst(aff);
4149 if (data->r >= 0 && data->r) {
4150 isl_aff_get_constant(aff, &data->t);
4151 if (data->first) {
4152 isl_int_set(data->c1, data->t);
4153 data->first = 0;
4154 } else if (!isl_int_eq(data->c1, data->t))
4155 data->equal = 0;
4156 } else if (data->r >= 0 && !data->r)
4157 data->equal = 0;
4159 isl_aff_free(aff);
4160 isl_set_free(set);
4161 isl_multi_aff_free(ma);
4163 if (data->r < 0)
4164 return -1;
4165 if (!data->equal)
4166 return -1;
4167 return 0;
4170 /* This function is called for each isl_pw_multi_aff in
4171 * the isl_union_pw_multi_aff checked by extract_int.
4172 * Check all the isl_multi_affs inside "pma".
4174 static int pw_multi_aff_extract_int(__isl_take isl_pw_multi_aff *pma,
4175 void *user)
4177 int r;
4179 r = isl_pw_multi_aff_foreach_piece(pma, &multi_aff_extract_int, user);
4180 isl_pw_multi_aff_free(pma);
4182 return r;
4185 /* Check if "upma" assigns a single constant value to its domain.
4186 * If so, return 1 and store the result in data->c1.
4187 * If not, return 0.
4189 * A negative return value from isl_union_pw_multi_aff_foreach_pw_multi_aff
4190 * means that either an error occurred or that we have broken off the check
4191 * because we already know the result is going to be negative.
4192 * In the latter case, data->equal is set to zero.
4194 static int extract_int(__isl_keep isl_union_pw_multi_aff *upma,
4195 struct isl_cmp_band_data *data)
4197 data->first = 1;
4198 data->equal = 1;
4200 if (isl_union_pw_multi_aff_foreach_pw_multi_aff(upma,
4201 &pw_multi_aff_extract_int, data) < 0) {
4202 if (!data->equal)
4203 return 0;
4204 return -1;
4207 return !data->first && data->equal;
4210 /* Compare "b1" and "b2" based on the parent schedule of their ancestor
4211 * "ancestor".
4213 * If the parent of "ancestor" also has a single member, then we
4214 * first try to compare the two band based on the partial schedule
4215 * of this parent.
4217 * Otherwise, or if the result is inconclusive, we look at the partial schedule
4218 * of "ancestor" itself.
4219 * In particular, we specialize the parent schedule based
4220 * on the domains of the child schedules, check if both assign
4221 * a single constant value and, if so, compare the two constant values.
4222 * If the specialized parent schedules do not assign a constant value,
4223 * then they cannot be used to order the two bands and so in this case
4224 * we return 0.
4226 static int cmp_band_in_ancestor(__isl_keep isl_band *b1,
4227 __isl_keep isl_band *b2, struct isl_cmp_band_data *data,
4228 __isl_keep isl_band *ancestor)
4230 isl_union_pw_multi_aff *upma;
4231 isl_union_set *domain;
4232 int r;
4234 if (data->r < 0)
4235 return 0;
4237 if (ancestor->parent && ancestor->parent->n == 1) {
4238 r = cmp_band_in_ancestor(b1, b2, data, ancestor->parent);
4239 if (data->r < 0)
4240 return 0;
4241 if (r)
4242 return r;
4245 upma = isl_union_pw_multi_aff_copy(b1->pma);
4246 domain = isl_union_pw_multi_aff_domain(upma);
4247 upma = isl_union_pw_multi_aff_copy(ancestor->pma);
4248 upma = isl_union_pw_multi_aff_intersect_domain(upma, domain);
4249 r = extract_int(upma, data);
4250 isl_union_pw_multi_aff_free(upma);
4252 if (r < 0)
4253 data->r = -1;
4254 if (r < 0 || !r)
4255 return 0;
4257 isl_int_set(data->c2, data->c1);
4259 upma = isl_union_pw_multi_aff_copy(b2->pma);
4260 domain = isl_union_pw_multi_aff_domain(upma);
4261 upma = isl_union_pw_multi_aff_copy(ancestor->pma);
4262 upma = isl_union_pw_multi_aff_intersect_domain(upma, domain);
4263 r = extract_int(upma, data);
4264 isl_union_pw_multi_aff_free(upma);
4266 if (r < 0)
4267 data->r = -1;
4268 if (r < 0 || !r)
4269 return 0;
4271 return isl_int_cmp(data->c2, data->c1);
4274 /* Compare "a" and "b" based on the parent schedule of their parent.
4276 static int cmp_band(const void *a, const void *b, void *user)
4278 isl_band *b1 = *(isl_band * const *) a;
4279 isl_band *b2 = *(isl_band * const *) b;
4280 struct isl_cmp_band_data *data = user;
4282 return cmp_band_in_ancestor(b1, b2, data, b1->parent);
4285 /* Sort the elements in "list" based on the partial schedules of its parent
4286 * (and ancestors). In particular if the parent assigns constant values
4287 * to the domains of the bands in "list", then the elements are sorted
4288 * according to that order.
4289 * This order should be a more "natural" order for the user, but otherwise
4290 * shouldn't have any effect.
4291 * If we would be constructing an isl_band forest directly in
4292 * isl_schedule_constraints_compute_schedule then there wouldn't be any need
4293 * for a reordering, since the children would be added to the list
4294 * in their natural order automatically.
4296 * If there is only one element in the list, then there is no need to sort
4297 * anything.
4298 * If the partial schedule of the parent has more than one member
4299 * (or if there is no parent), then it's
4300 * defnitely not assigning constant values to the different children in
4301 * the list and so we wouldn't be able to use it to sort the list.
4303 static __isl_give isl_band_list *sort_band_list(__isl_take isl_band_list *list,
4304 __isl_keep isl_band *parent)
4306 struct isl_cmp_band_data data;
4308 if (!list)
4309 return NULL;
4310 if (list->n <= 1)
4311 return list;
4312 if (!parent || parent->n != 1)
4313 return list;
4315 data.r = 0;
4316 isl_int_init(data.c1);
4317 isl_int_init(data.c2);
4318 isl_int_init(data.t);
4319 isl_sort(list->p, list->n, sizeof(list->p[0]), &cmp_band, &data);
4320 if (data.r < 0)
4321 list = isl_band_list_free(list);
4322 isl_int_clear(data.c1);
4323 isl_int_clear(data.c2);
4324 isl_int_clear(data.t);
4326 return list;
4329 /* Construct a list of bands that start at the same position (with
4330 * sequence number band_nr) in the schedules of the nodes that
4331 * were active in the parent band.
4333 * A separate isl_band structure is created for each band_id
4334 * and for each node that does not have a band with sequence
4335 * number band_nr. In the latter case, a band without members
4336 * is created.
4337 * This ensures that if a band has any children, then each node
4338 * that was active in the band is active in exactly one of the children.
4340 static __isl_give isl_band_list *construct_band_list(
4341 __isl_keep isl_schedule *schedule, __isl_keep isl_band *parent,
4342 int band_nr, int *parent_active, int n_active)
4344 int i, j;
4345 isl_ctx *ctx = isl_schedule_get_ctx(schedule);
4346 int *active;
4347 int n_band;
4348 isl_band_list *list;
4350 n_band = 0;
4351 for (i = 0; i < n_active; ++i) {
4352 for (j = 0; j < schedule->n; ++j) {
4353 if (!parent_active[j])
4354 continue;
4355 if (schedule->node[j].n_band <= band_nr)
4356 continue;
4357 if (schedule->node[j].band_id[band_nr] == i) {
4358 n_band++;
4359 break;
4363 for (j = 0; j < schedule->n; ++j)
4364 if (schedule->node[j].n_band <= band_nr)
4365 n_band++;
4367 if (n_band == 1) {
4368 isl_band *band;
4369 list = isl_band_list_alloc(ctx, n_band);
4370 band = construct_band(schedule, parent, band_nr,
4371 parent_active, n_active);
4372 return isl_band_list_add(list, band);
4375 active = isl_alloc_array(ctx, int, schedule->n);
4376 if (schedule->n && !active)
4377 return NULL;
4379 list = isl_band_list_alloc(ctx, n_band);
4381 for (i = 0; i < n_active; ++i) {
4382 int n = 0;
4383 isl_band *band;
4385 for (j = 0; j < schedule->n; ++j) {
4386 active[j] = parent_active[j] &&
4387 schedule->node[j].n_band > band_nr &&
4388 schedule->node[j].band_id[band_nr] == i;
4389 if (active[j])
4390 n++;
4392 if (n == 0)
4393 continue;
4395 band = construct_band(schedule, parent, band_nr, active, n);
4397 list = isl_band_list_add(list, band);
4399 for (i = 0; i < schedule->n; ++i) {
4400 isl_band *band;
4401 if (!parent_active[i])
4402 continue;
4403 if (schedule->node[i].n_band > band_nr)
4404 continue;
4405 for (j = 0; j < schedule->n; ++j)
4406 active[j] = j == i;
4407 band = construct_band(schedule, parent, band_nr, active, 1);
4408 list = isl_band_list_add(list, band);
4411 free(active);
4413 list = sort_band_list(list, parent);
4415 return list;
4418 /* Construct a band forest representation of the schedule and
4419 * return the list of roots.
4421 static __isl_give isl_band_list *construct_forest(
4422 __isl_keep isl_schedule *schedule)
4424 int i;
4425 isl_ctx *ctx = isl_schedule_get_ctx(schedule);
4426 isl_band_list *forest;
4427 int *active;
4429 active = isl_alloc_array(ctx, int, schedule->n);
4430 if (schedule->n && !active)
4431 return NULL;
4433 for (i = 0; i < schedule->n; ++i)
4434 active[i] = 1;
4436 forest = construct_band_list(schedule, NULL, 0, active, schedule->n);
4438 free(active);
4440 return forest;
4443 /* Return the roots of a band forest representation of the schedule.
4445 __isl_give isl_band_list *isl_schedule_get_band_forest(
4446 __isl_keep isl_schedule *schedule)
4448 if (!schedule)
4449 return NULL;
4450 if (!schedule->band_forest)
4451 schedule->band_forest = construct_forest(schedule);
4452 return isl_band_list_dup(schedule->band_forest);
4455 /* Call "fn" on each band in the schedule in depth-first post-order.
4457 int isl_schedule_foreach_band(__isl_keep isl_schedule *sched,
4458 int (*fn)(__isl_keep isl_band *band, void *user), void *user)
4460 int r;
4461 isl_band_list *forest;
4463 if (!sched)
4464 return -1;
4466 forest = isl_schedule_get_band_forest(sched);
4467 r = isl_band_list_foreach_band(forest, fn, user);
4468 isl_band_list_free(forest);
4470 return r;
4473 static __isl_give isl_printer *print_band_list(__isl_take isl_printer *p,
4474 __isl_keep isl_band_list *list);
4476 static __isl_give isl_printer *print_band(__isl_take isl_printer *p,
4477 __isl_keep isl_band *band)
4479 isl_band_list *children;
4481 p = isl_printer_start_line(p);
4482 p = isl_printer_print_union_pw_multi_aff(p, band->pma);
4483 p = isl_printer_end_line(p);
4485 if (!isl_band_has_children(band))
4486 return p;
4488 children = isl_band_get_children(band);
4490 p = isl_printer_indent(p, 4);
4491 p = print_band_list(p, children);
4492 p = isl_printer_indent(p, -4);
4494 isl_band_list_free(children);
4496 return p;
4499 static __isl_give isl_printer *print_band_list(__isl_take isl_printer *p,
4500 __isl_keep isl_band_list *list)
4502 int i, n;
4504 n = isl_band_list_n_band(list);
4505 for (i = 0; i < n; ++i) {
4506 isl_band *band;
4507 band = isl_band_list_get_band(list, i);
4508 p = print_band(p, band);
4509 isl_band_free(band);
4512 return p;
4515 __isl_give isl_printer *isl_printer_print_schedule(__isl_take isl_printer *p,
4516 __isl_keep isl_schedule *schedule)
4518 isl_band_list *forest;
4520 forest = isl_schedule_get_band_forest(schedule);
4522 p = print_band_list(p, forest);
4524 isl_band_list_free(forest);
4526 return p;
4529 void isl_schedule_dump(__isl_keep isl_schedule *schedule)
4531 isl_printer *printer;
4533 if (!schedule)
4534 return;
4536 printer = isl_printer_to_file(isl_schedule_get_ctx(schedule), stderr);
4537 printer = isl_printer_print_schedule(printer, schedule);
4539 isl_printer_free(printer);