return i915_request_completed(node_to_request(node));
}
-void i915_sched_node_init(struct i915_sched_node *node)
-{
- INIT_LIST_HEAD(&node->signalers_list);
- INIT_LIST_HEAD(&node->waiters_list);
- INIT_LIST_HEAD(&node->link);
- node->attr.priority = I915_PRIORITY_INVALID;
- node->semaphores = 0;
- node->flags = 0;
-}
-
-static struct i915_dependency *
-i915_dependency_alloc(void)
-{
- return kmem_cache_alloc(global.slab_dependencies, GFP_KERNEL);
-}
-
-static void
-i915_dependency_free(struct i915_dependency *dep)
-{
- kmem_cache_free(global.slab_dependencies, dep);
-}
-
-bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
- struct i915_sched_node *signal,
- struct i915_dependency *dep,
- unsigned long flags)
-{
- bool ret = false;
-
- spin_lock_irq(&schedule_lock);
-
- if (!node_signaled(signal)) {
- INIT_LIST_HEAD(&dep->dfs_link);
- list_add(&dep->wait_link, &signal->waiters_list);
- list_add(&dep->signal_link, &node->signalers_list);
- dep->signaler = signal;
- dep->flags = flags;
-
- /* Keep track of whether anyone on this chain has a semaphore */
- if (signal->flags & I915_SCHED_HAS_SEMAPHORE_CHAIN &&
- !node_started(signal))
- node->flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN;
-
- ret = true;
- }
-
- spin_unlock_irq(&schedule_lock);
-
- return ret;
-}
-
-int i915_sched_node_add_dependency(struct i915_sched_node *node,
- struct i915_sched_node *signal)
-{
- struct i915_dependency *dep;
-
- dep = i915_dependency_alloc();
- if (!dep)
- return -ENOMEM;
-
- if (!__i915_sched_node_add_dependency(node, signal, dep,
- I915_DEPENDENCY_ALLOC))
- i915_dependency_free(dep);
-
- return 0;
-}
-
-void i915_sched_node_fini(struct i915_sched_node *node)
-{
- struct i915_dependency *dep, *tmp;
-
- GEM_BUG_ON(!list_empty(&node->link));
-
- spin_lock_irq(&schedule_lock);
-
- /*
- * Everyone we depended upon (the fences we wait to be signaled)
- * should retire before us and remove themselves from our list.
- * However, retirement is run independently on each timeline and
- * so we may be called out-of-order.
- */
- list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) {
- GEM_BUG_ON(!node_signaled(dep->signaler));
- GEM_BUG_ON(!list_empty(&dep->dfs_link));
-
- list_del(&dep->wait_link);
- if (dep->flags & I915_DEPENDENCY_ALLOC)
- i915_dependency_free(dep);
- }
-
- /* Remove ourselves from everyone who depends upon us */
- list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link) {
- GEM_BUG_ON(dep->signaler != node);
- GEM_BUG_ON(!list_empty(&dep->dfs_link));
-
- list_del(&dep->signal_link);
- if (dep->flags & I915_DEPENDENCY_ALLOC)
- i915_dependency_free(dep);
- }
-
- spin_unlock_irq(&schedule_lock);
-}
-
static inline struct i915_priolist *to_priolist(struct rb_node *rb)
{
return rb_entry(rb, struct i915_priolist, node);
return &p->requests[idx];
}
+void __i915_priolist_free(struct i915_priolist *p)
+{
+ kmem_cache_free(global.slab_priorities, p);
+}
+
struct sched_cache {
struct list_head *priolist;
};
return active->hw_context == rq->hw_context;
}
-static void __i915_schedule(struct i915_request *rq,
+static void __i915_schedule(struct i915_sched_node *node,
const struct i915_sched_attr *attr)
{
struct intel_engine_cs *engine;
lockdep_assert_held(&schedule_lock);
GEM_BUG_ON(prio == I915_PRIORITY_INVALID);
- if (i915_request_completed(rq))
+ if (node_signaled(node))
return;
- if (prio <= READ_ONCE(rq->sched.attr.priority))
+ if (prio <= READ_ONCE(node->attr.priority))
return;
- stack.signaler = &rq->sched;
+ stack.signaler = node;
list_add(&stack.dfs_link, &dfs);
/*
* execlists_submit_request()), we can set our own priority and skip
* acquiring the engine locks.
*/
- if (rq->sched.attr.priority == I915_PRIORITY_INVALID) {
- GEM_BUG_ON(!list_empty(&rq->sched.link));
- rq->sched.attr = *attr;
+ if (node->attr.priority == I915_PRIORITY_INVALID) {
+ GEM_BUG_ON(!list_empty(&node->link));
+ node->attr = *attr;
if (stack.dfs_link.next == stack.dfs_link.prev)
return;
}
memset(&cache, 0, sizeof(cache));
- engine = rq->engine;
+ engine = node_to_request(node)->engine;
spin_lock(&engine->timeline.lock);
/* Fifo and depth-first replacement ensure our deps execute before us */
list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
- struct i915_sched_node *node = dep->signaler;
-
INIT_LIST_HEAD(&dep->dfs_link);
+ node = dep->signaler;
engine = sched_lock_engine(node, engine, &cache);
lockdep_assert_held(&engine->timeline.lock);
void i915_schedule(struct i915_request *rq, const struct i915_sched_attr *attr)
{
spin_lock_irq(&schedule_lock);
- __i915_schedule(rq, attr);
+ __i915_schedule(&rq->sched, attr);
spin_unlock_irq(&schedule_lock);
}
+static void __bump_priority(struct i915_sched_node *node, unsigned int bump)
+{
+ struct i915_sched_attr attr = node->attr;
+
+ attr.priority |= bump;
+ __i915_schedule(node, &attr);
+}
+
void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump)
{
- struct i915_sched_attr attr;
unsigned long flags;
GEM_BUG_ON(bump & ~I915_PRIORITY_MASK);
return;
spin_lock_irqsave(&schedule_lock, flags);
+ __bump_priority(&rq->sched, bump);
+ spin_unlock_irqrestore(&schedule_lock, flags);
+}
- attr = rq->sched.attr;
- attr.priority |= bump;
- __i915_schedule(rq, &attr);
+void i915_sched_node_init(struct i915_sched_node *node)
+{
+ INIT_LIST_HEAD(&node->signalers_list);
+ INIT_LIST_HEAD(&node->waiters_list);
+ INIT_LIST_HEAD(&node->link);
+ node->attr.priority = I915_PRIORITY_INVALID;
+ node->semaphores = 0;
+ node->flags = 0;
+}
- spin_unlock_irqrestore(&schedule_lock, flags);
+static struct i915_dependency *
+i915_dependency_alloc(void)
+{
+ return kmem_cache_alloc(global.slab_dependencies, GFP_KERNEL);
}
-void __i915_priolist_free(struct i915_priolist *p)
+static void
+i915_dependency_free(struct i915_dependency *dep)
{
- kmem_cache_free(global.slab_priorities, p);
+ kmem_cache_free(global.slab_dependencies, dep);
+}
+
+bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
+ struct i915_sched_node *signal,
+ struct i915_dependency *dep,
+ unsigned long flags)
+{
+ bool ret = false;
+
+ spin_lock_irq(&schedule_lock);
+
+ if (!node_signaled(signal)) {
+ INIT_LIST_HEAD(&dep->dfs_link);
+ list_add(&dep->wait_link, &signal->waiters_list);
+ list_add(&dep->signal_link, &node->signalers_list);
+ dep->signaler = signal;
+ dep->flags = flags;
+
+ /* Keep track of whether anyone on this chain has a semaphore */
+ if (signal->flags & I915_SCHED_HAS_SEMAPHORE_CHAIN &&
+ !node_started(signal))
+ node->flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN;
+
+ /*
+ * As we do not allow WAIT to preempt inflight requests,
+ * once we have executed a request, along with triggering
+ * any execution callbacks, we must preserve its ordering
+ * within the non-preemptible FIFO.
+ */
+ BUILD_BUG_ON(__NO_PREEMPTION & ~I915_PRIORITY_MASK);
+ if (flags & I915_DEPENDENCY_EXTERNAL)
+ __bump_priority(signal, __NO_PREEMPTION);
+
+ ret = true;
+ }
+
+ spin_unlock_irq(&schedule_lock);
+
+ return ret;
+}
+
+int i915_sched_node_add_dependency(struct i915_sched_node *node,
+ struct i915_sched_node *signal)
+{
+ struct i915_dependency *dep;
+
+ dep = i915_dependency_alloc();
+ if (!dep)
+ return -ENOMEM;
+
+ if (!__i915_sched_node_add_dependency(node, signal, dep,
+ I915_DEPENDENCY_EXTERNAL |
+ I915_DEPENDENCY_ALLOC))
+ i915_dependency_free(dep);
+
+ return 0;
+}
+
+void i915_sched_node_fini(struct i915_sched_node *node)
+{
+ struct i915_dependency *dep, *tmp;
+
+ GEM_BUG_ON(!list_empty(&node->link));
+
+ spin_lock_irq(&schedule_lock);
+
+ /*
+ * Everyone we depended upon (the fences we wait to be signaled)
+ * should retire before us and remove themselves from our list.
+ * However, retirement is run independently on each timeline and
+ * so we may be called out-of-order.
+ */
+ list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) {
+ GEM_BUG_ON(!node_signaled(dep->signaler));
+ GEM_BUG_ON(!list_empty(&dep->dfs_link));
+
+ list_del(&dep->wait_link);
+ if (dep->flags & I915_DEPENDENCY_ALLOC)
+ i915_dependency_free(dep);
+ }
+
+ /* Remove ourselves from everyone who depends upon us */
+ list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link) {
+ GEM_BUG_ON(dep->signaler != node);
+ GEM_BUG_ON(!list_empty(&dep->dfs_link));
+
+ list_del(&dep->signal_link);
+ if (dep->flags & I915_DEPENDENCY_ALLOC)
+ i915_dependency_free(dep);
+ }
+
+ spin_unlock_irq(&schedule_lock);
}
static void i915_global_scheduler_shrink(void)
projects / linux.git / blobdiff