diff --git a/internal/dnsforward/dnsforward.go b/internal/dnsforward/dnsforward.go
index 2f8b8c50..78777002 100644
--- a/internal/dnsforward/dnsforward.go
+++ b/internal/dnsforward/dnsforward.go
@@ -2,9 +2,11 @@
 package dnsforward
 
 import (
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"runtime"
 	"sync"
 	"time"
@@ -198,10 +200,16 @@ func (s *Server) Prepare(config *ServerConfig) error {
 	// --
 	err := s.ipset.init(s.conf.IPSETList)
 	if err != nil {
-		// ipset cannot be initialized in a Snap version (and maybe - without root)
-		// this needs to be handled properly
-		// TODO: Handle this properly
-		log.Info("Cannot initialize ipset module due to %v", err)
+		if !errors.Is(err, os.ErrPermission) {
+			return fmt.Errorf("cannot initialize ipset: %w", err)
+		}
+
+		// ipset cannot currently be initialized if the server was
+		// installed from Snap or when the user or the binary doesn't
+		// have the required permissions.
+		//
+		// Log and go on.
+		log.Error("cannot initialize ipset: %s", err)
 	}
 
 	// Prepare DNS servers settings
diff --git a/internal/dnsforward/ipset_linux.go b/internal/dnsforward/ipset_linux.go
index b62744f7..cf6a570f 100644
--- a/internal/dnsforward/ipset_linux.go
+++ b/internal/dnsforward/ipset_linux.go
@@ -37,6 +37,11 @@ type ipsetCtx struct {
 	nameToIpset    map[string]ipsetProps
 	domainToIpsets map[string][]ipsetProps
 
+	// TODO(a.garipov): Currently, the ipset list is static, and we don't
+	// read the IPs already in sets, so we can assume that all incoming IPs
+	// are either added to all corresponding ipsets or not.  When that stops
+	// being the case, for example if we add dynamic reconfiguration of
+	// ipsets, this map will need to become a per-ipset-name one.
 	addedIPs map[[16]byte]struct{}
 
 	ipv4Conn *ipset.Conn
@@ -289,10 +294,14 @@ func (c *ipsetCtx) skipIpsetProcessing(ctx *dnsContext) (ok bool) {
 
 // process adds the resolved IP addresses to the domain's ipsets, if any.
 func (c *ipsetCtx) process(ctx *dnsContext) (rc resultCode) {
+	var err error
+
 	if c == nil {
 		return resultCodeSuccess
 	}
 
+	log.Debug("ipset: starting processing")
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -308,6 +317,8 @@ func (c *ipsetCtx) process(ctx *dnsContext) (rc resultCode) {
 	host = strings.ToLower(host)
 	sets := c.lookupHost(host)
 	if len(sets) == 0 {
+		log.Debug("ipset: no ipsets for host %s", host)
+
 		return resultCodeSuccess
 	}
 
@@ -342,7 +353,6 @@ func (c *ipsetCtx) process(ctx *dnsContext) (rc resultCode) {
 		v4s = append(v4s, ip)
 	}
 
-	var err error
 setLoop:
 	for _, set := range sets {
 		switch set.family {
@@ -363,6 +373,20 @@ setLoop:
 	}
 	if err != nil {
 		log.Error("ipset: adding host ips: %s", err)
+	} else {
+		log.Debug("ipset: processed %d new ips", len(v4s)+len(v6s))
+	}
+
+	for _, ip := range v4s {
+		var iparr [16]byte
+		copy(iparr[:], ip.To16())
+		c.addedIPs[iparr] = struct{}{}
+	}
+
+	for _, ip := range v6s {
+		var iparr [16]byte
+		copy(iparr[:], ip.To16())
+		c.addedIPs[iparr] = struct{}{}
 	}
 
 	return resultCodeSuccess